diff options
37 files changed, 611 insertions, 554 deletions
diff --git a/rhodecode/config/conf.py b/rhodecode/config/conf.py new file mode 100644 index 00000000..1e699ae9 --- /dev/null +++ b/rhodecode/config/conf.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" + package.rhodecode.config.conf + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Various config settings for RhodeCode + + :created_on: Mar 7, 2012 + :author: marcink + :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> + :license: <name>, see LICENSE_FILE for more details. +""" +from rhodecode import EXTENSIONS + +from rhodecode.lib.utils2 import __get_lem + + +# language map is also used by whoosh indexer, which for those specified +# extensions will index it's content +LANGUAGES_EXTENSIONS_MAP = __get_lem() + +#============================================================================== +# WHOOSH INDEX EXTENSIONS +#============================================================================== +# EXTENSIONS WE WANT TO INDEX CONTENT OFF USING WHOOSH +INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys() + +# list of readme files to search in file tree and display in summary +# attached weights defines the search order lower is first +ALL_READMES = [ + ('readme', 0), ('README', 0), ('Readme', 0), + ('doc/readme', 1), ('doc/README', 1), ('doc/Readme', 1), + ('Docs/readme', 2), ('Docs/README', 2), ('Docs/Readme', 2), + ('DOCS/readme', 2), ('DOCS/README', 2), ('DOCS/Readme', 2), + ('docs/readme', 2), ('docs/README', 2), ('docs/Readme', 2), +] + +# extension together with weights to search lower is first +RST_EXTS = [ + ('', 0), ('.rst', 1), ('.rest', 1), + ('.RST', 2), ('.REST', 2), + ('.txt', 3), ('.TXT', 3) +] + +MARKDOWN_EXTS = [ + ('.md', 1), ('.MD', 1), + ('.mkdn', 2), ('.MKDN', 2), + ('.mdown', 3), ('.MDOWN', 3), + ('.markdown', 4), ('.MARKDOWN', 4) +] + +PLAIN_EXTS = [('.text', 2), ('.TEXT', 2)] + +ALL_EXTS = MARKDOWN_EXTS + RST_EXTS + PLAIN_EXTS + +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" + +DATE_FORMAT = "%Y-%m-%d" diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py index 1217c3fe..470a92f5 100644 --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -2,40 +2,45 @@ import os import logging +import rhodecode from mako.lookup import TemplateLookup from pylons.configuration import PylonsConfig from pylons.error import handle_mako_error -import rhodecode +# don't remove this import it does magic for celery +from rhodecode.lib import celerypylons + import rhodecode.lib.app_globals as app_globals -import rhodecode.lib.helpers from rhodecode.config.routing import make_map -# don't remove this import it does magic for celery -from rhodecode.lib import celerypylons, str2bool -from rhodecode.lib import engine_from_config + +from rhodecode.lib import helpers from rhodecode.lib.auth import set_available_permissions -from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config +from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config,\ + load_rcextensions +from rhodecode.lib.utils2 import engine_from_config, str2bool from rhodecode.model import init_model from rhodecode.model.scm import ScmModel -from rhodecode.lib.vcs.utils.fakemod import create_module log = logging.getLogger(__name__) def load_environment(global_conf, app_conf, initial=False): - """Configure the Pylons environment via the ``pylons.config`` + """ + Configure the Pylons environment via the ``pylons.config`` object """ config = PylonsConfig() # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - paths = dict(root=root, - controllers=os.path.join(root, 'controllers'), - static_files=os.path.join(root, 'public'), - templates=[os.path.join(root, 'templates')]) + paths = dict( + root=root, + controllers=os.path.join(root, 'controllers'), + static_files=os.path.join(root, 'public'), + templates=[os.path.join(root, 'templates')] + ) # Initialize config with the basic options config.init_app(global_conf, app_conf, package='rhodecode', paths=paths) @@ -45,14 +50,11 @@ def load_environment(global_conf, app_conf, initial=False): config['routes.map'] = make_map(config) config['pylons.app_globals'] = app_globals.Globals(config) - config['pylons.h'] = rhodecode.lib.helpers + config['pylons.h'] = helpers rhodecode.CONFIG = config - path = os.path.join(config['here'], 'rcextensions', '__init__.py') - if os.path.isfile(path): - rcext = create_module('rc', path) - rhodecode.EXTENSIONS = rcext - log.debug('Found rcextensions now loading %s...' % rcext) + load_rcextensions(root_path=config['here']) + # Setup cache object as early as possible import pylons pylons.cache._push_object(config['pylons.app_globals'].cache) diff --git a/rhodecode/config/rcextensions/make_rcextensions.py b/rhodecode/config/rcextensions/make_rcextensions.py index 28a251ce..fb57fade 100644 --- a/rhodecode/config/rcextensions/make_rcextensions.py +++ b/rhodecode/config/rcextensions/make_rcextensions.py @@ -77,5 +77,3 @@ class MakeRcExt(BasePasterCommand): def update_parser(self): pass - - diff --git a/rhodecode/controllers/admin/users_groups.py b/rhodecode/controllers/admin/users_groups.py index 04193bcb..b47d6e7a 100644 --- a/rhodecode/controllers/admin/users_groups.py +++ b/rhodecode/controllers/admin/users_groups.py @@ -32,8 +32,9 @@ from pylons import request, session, tmpl_context as c, url, config from pylons.controllers.util import abort, redirect from pylons.i18n.translation import _ +from rhodecode.lib import helpers as h from rhodecode.lib.exceptions import UsersGroupsAssignedException -from rhodecode.lib import helpers as h, safe_unicode +from rhodecode.lib.utils2 import safe_unicode from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator from rhodecode.lib.base import BaseController, render diff --git a/rhodecode/controllers/branches.py b/rhodecode/controllers/branches.py index 93a324a8..eead9f51 100644 --- a/rhodecode/controllers/branches.py +++ b/rhodecode/controllers/branches.py @@ -31,7 +31,7 @@ import binascii from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseRepoController, render from rhodecode.lib.compat import OrderedDict -from rhodecode.lib import safe_unicode +from rhodecode.lib.utils2 import safe_unicode log = logging.getLogger(__name__) diff --git a/rhodecode/controllers/feed.py b/rhodecode/controllers/feed.py index 093cd257..1c15c7cf 100644 --- a/rhodecode/controllers/feed.py +++ b/rhodecode/controllers/feed.py @@ -28,7 +28,7 @@ import logging from pylons import url, response, tmpl_context as c from pylons.i18n.translation import _ -from rhodecode.lib import safe_unicode +from rhodecode.lib.utils2 import safe_unicode from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseRepoController diff --git a/rhodecode/controllers/files.py b/rhodecode/controllers/files.py index c49bb07a..bb2d5af0 100644 --- a/rhodecode/controllers/files.py +++ b/rhodecode/controllers/files.py @@ -32,24 +32,26 @@ from pylons.i18n.translation import _ from pylons.controllers.util import redirect from pylons.decorators import jsonify -from rhodecode.lib.vcs.conf import settings -from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \ - EmptyRepositoryError, ImproperArchiveTypeError, VCSError, \ - NodeAlreadyExistsError -from rhodecode.lib.vcs.nodes import FileNode +from rhodecode.lib import diffs +from rhodecode.lib import helpers as h from rhodecode.lib.compat import OrderedDict -from rhodecode.lib import convert_line_endings, detect_mode, safe_str +from rhodecode.lib.utils2 import convert_line_endings, detect_mode, safe_str from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseRepoController, render from rhodecode.lib.utils import EmptyChangeset -from rhodecode.lib import diffs -import rhodecode.lib.helpers as h +from rhodecode.lib.vcs.conf import settings +from rhodecode.lib.vcs.exceptions import RepositoryError, \ + ChangesetDoesNotExistError, EmptyRepositoryError, \ + ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError +from rhodecode.lib.vcs.nodes import FileNode + from rhodecode.model.repo import RepoModel +from rhodecode.model.scm import ScmModel + from rhodecode.controllers.changeset import anchor_url, _ignorews_url,\ _context_url, get_line_ctx, get_ignore_ws -from rhodecode.lib.diffs import wrapped_diff -from rhodecode.model.scm import ScmModel + log = logging.getLogger(__name__) @@ -447,7 +449,7 @@ class FilesController(BaseRepoController): ign_whitespace_lcl = get_ignore_ws(fid, request.GET) lim = request.GET.get('fulldiff') or self.cut_off_limit - _, cs1, cs2, diff, st = wrapped_diff(filenode_old=node1, + _, cs1, cs2, diff, st = diffs.wrapped_diff(filenode_old=node1, filenode_new=node2, cut_off_limit=lim, ignore_whitespace=ign_whitespace_lcl, diff --git a/rhodecode/controllers/summary.py b/rhodecode/controllers/summary.py index 0a9f33f6..8b380232 100644 --- a/rhodecode/controllers/summary.py +++ b/rhodecode/controllers/summary.py @@ -40,15 +40,15 @@ from pylons.i18n.translation import _ from beaker.cache import cache_region, region_invalidate +from rhodecode.config.conf import ALL_READMES, ALL_EXTS, LANGUAGES_EXTENSIONS_MAP from rhodecode.model.db import Statistics, CacheInvalidation -from rhodecode.lib import ALL_READMES, ALL_EXTS, safe_unicode +from rhodecode.lib.utils2 import safe_unicode from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseRepoController, render from rhodecode.lib.utils import EmptyChangeset from rhodecode.lib.markup_renderer import MarkupRenderer from rhodecode.lib.celerylib import run_task -from rhodecode.lib.celerylib.tasks import get_commits_stats, \ - LANGUAGES_EXTENSIONS_MAP +from rhodecode.lib.celerylib.tasks import get_commits_stats from rhodecode.lib.helpers import RepoPage from rhodecode.lib.compat import json, OrderedDict diff --git a/rhodecode/lib/__init__.py b/rhodecode/lib/__init__.py index 4b341059..d0db1d07 100644 --- a/rhodecode/lib/__init__.py +++ b/rhodecode/lib/__init__.py @@ -1,444 +1,4 @@ -# -*- coding: utf-8 -*- -""" - rhodecode.lib.__init__ - ~~~~~~~~~~~~~~~~~~~~~~~ - - Some simple helper functions - - :created_on: Jan 5, 2011 - :author: marcink - :copyright: (C) 2011-2012 Marcin Kuzminski <marcin@python-works.com> - :license: GPLv3, see COPYING for more details. -""" -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - import os -import re -from rhodecode import EXTENSIONS -from rhodecode.lib.vcs.utils.lazy import LazyProperty - - -def __get_lem(): - from pygments import lexers - from string import lower - from collections import defaultdict - - d = defaultdict(lambda: []) - - def __clean(s): - s = s.lstrip('*') - s = s.lstrip('.') - - if s.find('[') != -1: - exts = [] - start, stop = s.find('['), s.find(']') - - for suffix in s[start + 1:stop]: - exts.append(s[:s.find('[')] + suffix) - return map(lower, exts) - else: - return map(lower, [s]) - - for lx, t in sorted(lexers.LEXERS.items()): - m = map(__clean, t[-2]) - if m: - m = reduce(lambda x, y: x + y, m) - for ext in m: - desc = lx.replace('Lexer', '') - d[ext].append(desc) - - return dict(d) - -# language map is also used by whoosh indexer, which for those specified -# extensions will index it's content -LANGUAGES_EXTENSIONS_MAP = __get_lem() - -# Additional mappings that are not present in the pygments lexers -LANGUAGES_EXTENSIONS_MAP.update(getattr(EXTENSIONS, 'EXTRA_MAPPINGS', {})) - -#============================================================================== -# WHOOSH INDEX EXTENSIONS -#============================================================================== -# EXTENSIONS WE WANT TO INDEX CONTENT OFF USING WHOOSH -INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys() - -#OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present) - -if getattr(EXTENSIONS, 'INDEX_EXTENSIONS', []) != []: - INDEX_EXTENSIONS = getattr(EXTENSIONS, 'INDEX_EXTENSIONS', []) - -#ADDITIONAL MAPPINGS -INDEX_EXTENSIONS.extend(getattr(EXTENSIONS, 'EXTRA_INDEX_EXTENSIONS', [])) - -# list of readme files to search in file tree and display in summary -# attached weights defines the search order lower is first -ALL_READMES = [ - ('readme', 0), ('README', 0), ('Readme', 0), - ('doc/readme', 1), ('doc/README', 1), ('doc/Readme', 1), - ('Docs/readme', 2), ('Docs/README', 2), ('Docs/Readme', 2), - ('DOCS/readme', 2), ('DOCS/README', 2), ('DOCS/Readme', 2), - ('docs/readme', 2), ('docs/README', 2), ('docs/Readme', 2), -] - -# extension together with weights to search lower is first -RST_EXTS = [ - ('', 0), ('.rst', 1), ('.rest', 1), - ('.RST', 2), ('.REST', 2), - ('.txt', 3), ('.TXT', 3) -] - -MARKDOWN_EXTS = [ - ('.md', 1), ('.MD', 1), - ('.mkdn', 2), ('.MKDN', 2), - ('.mdown', 3), ('.MDOWN', 3), - ('.markdown', 4), ('.MARKDOWN', 4) -] - -PLAIN_EXTS = [('.text', 2), ('.TEXT', 2)] - -ALL_EXTS = MARKDOWN_EXTS + RST_EXTS + PLAIN_EXTS - - -def str2bool(_str): - """ - returs True/False value from given string, it tries to translate the - string into boolean - - :param _str: string value to translate into boolean - :rtype: boolean - :returns: boolean from given string - """ - if _str is None: - return False - if _str in (True, False): - return _str - _str = str(_str).strip().lower() - return _str in ('t', 'true', 'y', 'yes', 'on', '1') - - -def convert_line_endings(line, mode): - """ - Converts a given line "line end" accordingly to given mode - - Available modes are:: - 0 - Unix - 1 - Mac - 2 - DOS - - :param line: given line to convert - :param mode: mode to convert to - :rtype: str - :return: converted line according to mode - """ - from string import replace - - if mode == 0: - line = replace(line, '\r\n', '\n') - line = replace(line, '\r', '\n') - elif mode == 1: - line = replace(line, '\r\n', '\r') - line = replace(line, '\n', '\r') - elif mode == 2: - line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line) - return line - - -def detect_mode(line, default): - """ - Detects line break for given line, if line break couldn't be found - given default value is returned - - :param line: str line - :param default: default - :rtype: int - :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS - """ - if line.endswith('\r\n'): - return 2 - elif line.endswith('\n'): - return 0 - elif line.endswith('\r'): - return 1 - else: - return default - - -def generate_api_key(username, salt=None): - """ - Generates unique API key for given username, if salt is not given - it'll be generated from some random string - - :param username: username as string - :param salt: salt to hash generate KEY - :rtype: str - :returns: sha1 hash from username+salt - """ - from tempfile import _RandomNameSequence - import hashlib - - if salt is None: - salt = _RandomNameSequence().next() - - return hashlib.sha1(username + salt).hexdigest() - - -def safe_unicode(str_, from_encoding=None): - """ - safe unicode function. Does few trick to turn str_ into unicode - - In case of UnicodeDecode error we try to return it with encoding detected - by chardet library if it fails fallback to unicode with errors replaced - - :param str_: string to decode - :rtype: unicode - :returns: unicode object - """ - if isinstance(str_, unicode): - return str_ - - if not from_encoding: - import rhodecode - DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8') - from_encoding = DEFAULT_ENCODING - - try: - return unicode(str_) - except UnicodeDecodeError: - pass - - try: - return unicode(str_, from_encoding) - except UnicodeDecodeError: - pass - - try: - import chardet - encoding = chardet.detect(str_)['encoding'] - if encoding is None: - raise Exception() - return str_.decode(encoding) - except (ImportError, UnicodeDecodeError, Exception): - return unicode(str_, from_encoding, 'replace') - - -def safe_str(unicode_, to_encoding=None): - """ - safe str function. Does few trick to turn unicode_ into string - - In case of UnicodeEncodeError we try to return it with encoding detected - by chardet library if it fails fallback to string with errors replaced - - :param unicode_: unicode to encode - :rtype: str - :returns: str object - """ - - # if it's not basestr cast to str - if not isinstance(unicode_, basestring): - return str(unicode_) - - if isinstance(unicode_, str): - return unicode_ - - if not to_encoding: - import rhodecode - DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8') - to_encoding = DEFAULT_ENCODING - - try: - return unicode_.encode(to_encoding) - except UnicodeEncodeError: - pass - - try: - import chardet - encoding = chardet.detect(unicode_)['encoding'] - print encoding - if encoding is None: - raise UnicodeEncodeError() - - return unicode_.encode(encoding) - except (ImportError, UnicodeEncodeError): - return unicode_.encode(to_encoding, 'replace') - - return safe_str - - -def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): - """ - Custom engine_from_config functions that makes sure we use NullPool for - file based sqlite databases. This prevents errors on sqlite. This only - applies to sqlalchemy versions < 0.7.0 - - """ - import sqlalchemy - from sqlalchemy import engine_from_config as efc - import logging - - if int(sqlalchemy.__version__.split('.')[1]) < 7: - - # This solution should work for sqlalchemy < 0.7.0, and should use - # proxy=TimerProxy() for execution time profiling - - from sqlalchemy.pool import NullPool - url = configuration[prefix + 'url'] - - if url.startswith('sqlite'): - kwargs.update({'poolclass': NullPool}) - return efc(configuration, prefix, **kwargs) - else: - import time - from sqlalchemy import event - from sqlalchemy.engine import Engine - - log = logging.getLogger('sqlalchemy.engine') - BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38) - engine = efc(configuration, prefix, **kwargs) - - def color_sql(sql): - COLOR_SEQ = "\033[1;%dm" - COLOR_SQL = YELLOW - normal = '\x1b[0m' - return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal]) - - if configuration['debug']: - #attach events only for debug configuration - - def before_cursor_execute(conn, cursor, statement, - parameters, context, executemany): - context._query_start_time = time.time() - log.info(color_sql(">>>>> STARTING QUERY >>>>>")) - - - def after_cursor_execute(conn, cursor, statement, - parameters, context, executemany): - total = time.time() - context._query_start_time - log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total)) - - event.listen(engine, "before_cursor_execute", - before_cursor_execute) - event.listen(engine, "after_cursor_execute", - after_cursor_execute) - - return engine - - -def age(curdate): - """ - turns a datetime into an age string. - - :param curdate: datetime object - :rtype: unicode - :returns: unicode words describing age - """ - - from datetime import datetime - from webhelpers.date import time_ago_in_words - - _ = lambda s: s - - if not curdate: - return '' - - agescales = [(_(u"year"), 3600 * 24 * 365), - (_(u"month"), 3600 * 24 * 30), - (_(u"day"), 3600 * 24), - (_(u"hour"), 3600), - (_(u"minute"), 60), - (_(u"second"), 1), ] - - age = datetime.now() - curdate - age_seconds = (age.days * agescales[2][1]) + age.seconds - pos = 1 - for scale in agescales: - if scale[1] <= age_seconds: - if pos == 6: - pos = 5 - return '%s %s' % (time_ago_in_words(curdate, - agescales[pos][0]), _('ago')) - pos += 1 - - return _(u'just now') - - -def uri_filter(uri): - """ - Removes user:password from given url string - - :param uri: - :rtype: unicode - :returns: filtered list of strings - """ - if not uri: - return '' - - proto = '' - - for pat in ('https://', 'http://'): - if uri.startswith(pat): - uri = uri[len(pat):] - proto = pat - break - - # remove passwords and username - uri = uri[uri.find('@') + 1:] - - # get the port - cred_pos = uri.find(':') - if cred_pos == -1: - host, port = uri, None - else: - host, port = uri[:cred_pos], uri[cred_pos + 1:] - - return filter(None, [proto, host, port]) - - -def credentials_filter(uri): - """ - Returns a url with removed credentials - - :param uri: - """ - - uri = uri_filter(uri) - #check if we have port - if len(uri) > 2 and uri[2]: - uri[2] = ':' + uri[2] - - return ''.join(uri) - - -def get_changeset_safe(repo, rev): - """ - Safe version of get_changeset if this changeset doesn't exists for a - repo it returns a Dummy one instead - - :param repo: - :param rev: - """ - from rhodecode.lib.vcs.backends.base import BaseRepository - from rhodecode.lib.vcs.exceptions import RepositoryError - if not isinstance(repo, BaseRepository): - raise Exception('You must pass an Repository ' - 'object as first argument got %s', type(repo)) - - try: - cs = repo.get_changeset(rev) - except RepositoryError: - from rhodecode.lib.utils import EmptyChangeset - cs = EmptyChangeset(requested_revision=rev) - return cs def get_current_revision(quiet=False): @@ -462,16 +22,3 @@ def get_current_revision(quiet=False): print ("Cannot retrieve rhodecode's revision. Original error " "was: %s" % err) return None - - -def extract_mentioned_users(s): - """ - Returns unique usernames from given string s that have @mention - - :param s: string to get mentions - """ - usrs = {} - for username in re.findall(r'(?:^@|\s@)(\w+)', s): - usrs[username] = username - - return sorted(usrs.keys()) diff --git a/rhodecode/lib/auth.py b/rhodecode/lib/auth.py index dbcd7f6a..a1cfe5b4 100644 --- a/rhodecode/lib/auth.py +++ b/rhodecode/lib/auth.py @@ -43,7 +43,7 @@ if __platform__ in PLATFORM_WIN: if __platform__ in PLATFORM_OTHERS: import bcrypt -from rhodecode.lib import str2bool, safe_unicode +from rhodecode.lib.utils2 import str2bool, safe_unicode from rhodecode.lib.exceptions import LdapPasswordError, LdapUsernameError from rhodecode.lib.utils import get_repo_slug, get_repos_group_slug from rhodecode.lib.auth_ldap import AuthLdap @@ -795,7 +795,7 @@ class HasPermissionAnyMiddleware(object): try: self.user_perms = set([usr.permissions['repositories'][repo_name]]) except Exception: - log.error('Exception while accessing permissions %s' % + log.error('Exception while accessing permissions %s' % traceback.format_exc()) self.user_perms = set() self.granted_for = '' diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py index 4698234e..27de031a 100644 --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -15,7 +15,7 @@ from pylons.templating import render_mako as render from rhodecode import __version__, BACKENDS -from rhodecode.lib import str2bool, safe_unicode +from rhodecode.lib.utils2 import str2bool, safe_unicode from rhodecode.lib.auth import AuthUser, get_container_username, authfunc,\ HasPermissionAnyMiddleware, CookieStoreWrapper from rhodecode.lib.utils import get_repo_slug, invalidate_cache diff --git a/rhodecode/lib/caching_query.py b/rhodecode/lib/caching_query.py index 00d3169c..ffa21c02 100644 --- a/rhodecode/lib/caching_query.py +++ b/rhodecode/lib/caching_query.py @@ -24,7 +24,7 @@ from beaker.exceptions import BeakerException from sqlalchemy.orm.interfaces import MapperOption from sqlalchemy.orm.query import Query from sqlalchemy.sql import visitors -from rhodecode.lib import safe_str +from rhodecode.lib.utils2 import safe_str class CachingQuery(Query): diff --git a/rhodecode/lib/celerylib/__init__.py b/rhodecode/lib/celerylib/__init__.py index c2287649..926f2bf5 100644 --- a/rhodecode/lib/celerylib/__init__.py +++ b/rhodecode/lib/celerylib/__init__.py @@ -36,7 +36,7 @@ from decorator import decorator from rhodecode.lib.vcs.utils.lazy import LazyProperty from rhodecode import CELERY_ON -from rhodecode.lib import str2bool, safe_str +from rhodecode.lib.utils2 import str2bool, safe_str from rhodecode.lib.pidlock import DaemonLock, LockHeld from rhodecode.model import init_model from rhodecode.model import meta diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py index adbb02e0..9668abf5 100644 --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -40,7 +40,7 @@ from pylons.i18n.translation import _ from rhodecode.lib.vcs import get_backend from rhodecode import CELERY_ON -from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, safe_str +from rhodecode.lib.utils2 import safe_str from rhodecode.lib.celerylib import run_task, locked_task, dbsession, \ str2bool, __get_lockkey, LockHeld, DaemonLock, get_session from rhodecode.lib.helpers import person @@ -147,6 +147,7 @@ def get_commits_stats(repo_name, ts_min_y, ts_max_y): last_rev, last_rev + parse_limit) ) for cs in repo[last_rev:last_rev + parse_limit]: + log.debug('parsing %s' % cs) last_cs = cs # remember last parsed changeset k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1], cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0]) @@ -233,10 +234,10 @@ def get_commits_stats(repo_name, ts_min_y, ts_max_y): lock.release() return False - #final release + # final release lock.release() - #execute another task if celery is enabled + # execute another task if celery is enabled if len(repo.revisions) > 1 and CELERY_ON: run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) return True @@ -395,6 +396,7 @@ def create_repo_fork(form_data, cur_user): DBS.commit() def __get_codes_stats(repo_name): + from rhodecode.config.conf import LANGUAGES_EXTENSIONS_MAP repo = Repository.get_by_repo_name(repo_name).scm_instance tip = repo.get_changeset() diff --git a/rhodecode/lib/celerypylons/commands.py b/rhodecode/lib/celerypylons/commands.py index c8fc12f7..15cf3001 100644 --- a/rhodecode/lib/celerypylons/commands.py +++ b/rhodecode/lib/celerypylons/commands.py @@ -1,9 +1,9 @@ import rhodecode -from rhodecode.lib.utils import BasePasterCommand, Command +from rhodecode.lib.utils import BasePasterCommand, Command, load_rcextensions from celery.app import app_or_default from celery.bin import camqadm, celerybeat, celeryd, celeryev -from rhodecode.lib import str2bool +from rhodecode.lib.utils2 import str2bool __all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand', 'CAMQPAdminCommand', 'CeleryEventCommand'] @@ -39,9 +39,11 @@ class CeleryCommand(BasePasterCommand): raise Exception('Please enable celery_on in .ini config ' 'file before running celeryd') rhodecode.CELERY_ON = CELERY_ON + load_rcextensions(config['here']) cmd = self.celery_command(app_or_default()) return cmd.run(**vars(self.options)) + class CeleryDaemonCommand(CeleryCommand): """Start the celery worker @@ -82,6 +84,7 @@ class CAMQPAdminCommand(CeleryCommand): parser = Command.standard_parser(quiet=True) celery_command = camqadm.AMQPAdminCommand + class CeleryEventCommand(CeleryCommand): """Celery event command. diff --git a/rhodecode/lib/dbmigrate/migrate/exceptions.py b/rhodecode/lib/dbmigrate/migrate/exceptions.py index cb8c4094..29b5850f 100644 --- a/rhodecode/lib/dbmigrate/migrate/exceptions.py +++ b/rhodecode/lib/dbmigrate/migrate/exceptions.py @@ -71,9 +71,6 @@ class InvalidScriptError(ScriptError): """Invalid script error.""" -class InvalidVersionError(Error): - """Invalid version error.""" - # migrate.changeset class NotSupportedError(Error): diff --git a/rhodecode/lib/dbmigrate/schema/db_1_2_0.py b/rhodecode/lib/dbmigrate/schema/db_1_2_0.py index deda56bd..5eb5cdbd 100755 --- a/rhodecode/lib/dbmigrate/schema/db_1_2_0.py +++ b/rhodecode/lib/dbmigrate/schema/db_1_2_0.py @@ -39,7 +39,7 @@ from rhodecode.lib.vcs.utils.helpers import get_scm from rhodecode.lib.vcs.exceptions import VCSError from rhodecode.lib.vcs.utils.lazy import LazyProperty -from rhodecode.lib import str2bool, safe_str, get_changeset_safe, \ +from rhodecode.lib.utils2 import str2bool, safe_str, get_changeset_safe, \ generate_api_key, safe_unicode from rhodecode.lib.exceptions import UsersGroupsAssignedException from rhodecode.lib.compat import json @@ -717,7 +717,7 @@ class Repository(Base, BaseModel): return repo -class RepoGroup(Base, BaseModel): +class Group(Base, BaseModel): __tablename__ = 'groups' __table_args__ = (UniqueConstraint('group_name', 'group_parent_id'), CheckConstraint('group_id != group_parent_id'), {'extend_existing':True},) @@ -728,8 +728,7 @@ class RepoGroup(Base, BaseModel): group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None) group_description = Column("group_description", String(length=10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) - parent_group = relationship('RepoGroup', remote_side=group_id) - + parent_group = relationship('Group', remote_side=group_id) def __init__(self, group_name='', parent_group=None): self.group_name = group_name diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py index 56550638..0dfae877 100644 --- a/rhodecode/lib/helpers.py +++ b/rhodecode/lib/helpers.py @@ -39,7 +39,8 @@ from webhelpers.html.tags import _set_input_attrs, _set_id_attr, \ from rhodecode.lib.annotate import annotate_highlight from rhodecode.lib.utils import repo_name_slug -from rhodecode.lib import str2bool, safe_unicode, safe_str, get_changeset_safe +from rhodecode.lib.utils2 import str2bool, safe_unicode, safe_str, \ + get_changeset_safe from rhodecode.lib.markup_renderer import MarkupRenderer log = logging.getLogger(__name__) @@ -319,7 +320,7 @@ flash = _Flash() # SCM FILTERS available via h. #============================================================================== from rhodecode.lib.vcs.utils import author_name, author_email -from rhodecode.lib import credentials_filter, age as _age +from rhodecode.lib.utils2 import credentials_filter, age as _age from rhodecode.model.db import User age = lambda x: _age(x) diff --git a/rhodecode/lib/hooks.py b/rhodecode/lib/hooks.py index b1cf9054..2be1c1b0 100644 --- a/rhodecode/lib/hooks.py +++ b/rhodecode/lib/hooks.py @@ -139,7 +139,7 @@ def log_push_action(ui, repo, **kwargs): def log_create_repository(repository_dict, created_by, **kwargs): """ Post create repository Hook. This is a dummy function for admins to re-use - if needed. It's taken from rhodecode-extensions module and executed + if needed. It's taken from rhodecode-extensions module and executed if present :param repository: dict dump of repository object diff --git a/rhodecode/lib/indexers/__init__.py b/rhodecode/lib/indexers/__init__.py index 925e614d..fa11f90b 100644 --- a/rhodecode/lib/indexers/__init__.py +++ b/rhodecode/lib/indexers/__init__.py @@ -47,9 +47,9 @@ from rhodecode.model import init_model from rhodecode.model.scm import ScmModel from rhodecode.model.repo import RepoModel from rhodecode.config.environment import load_environment -from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, INDEX_EXTENSIONS, \ - LazyProperty -from rhodecode.lib.utils import BasePasterCommand, Command, add_cache +from rhodecode.lib.utils2 import LazyProperty +from rhodecode.lib.utils import BasePasterCommand, Command, add_cache,\ + load_rcextensions # CUSTOM ANALYZER wordsplit + lowercase filter ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() @@ -88,13 +88,12 @@ class MakeIndex(BasePasterCommand): add_cache(config) engine = engine_from_config(config, 'sqlalchemy.db1.') init_model(engine) - index_location = config['index_dir'] repo_location = self.options.repo_location \ if self.options.repo_location else RepoModel().repos_path repo_list = map(strip, self.options.repo_list.split(',')) \ if self.options.repo_list else None - + load_rcextensions(config['here']) #====================================================================== # WHOOSH DAEMON #====================================================================== @@ -104,7 +103,7 @@ class MakeIndex(BasePasterCommand): l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock')) WhooshIndexingDaemon(index_location=index_location, repo_location=repo_location, - repo_list=repo_list)\ + repo_list=repo_list,)\ .run(full_index=self.options.full_index) l.release() except LockHeld: diff --git a/rhodecode/lib/indexers/daemon.py b/rhodecode/lib/indexers/daemon.py index d80a5c1a..9b5aa806 100644 --- a/rhodecode/lib/indexers/daemon.py +++ b/rhodecode/lib/indexers/daemon.py @@ -38,10 +38,10 @@ from os.path import join as jn project_path = dn(dn(dn(dn(os.path.realpath(__file__))))) sys.path.append(project_path) - +from rhodecode.config.conf import INDEX_EXTENSIONS from rhodecode.model.scm import ScmModel -from rhodecode.lib import safe_unicode -from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME +from rhodecode.lib.utils2 import safe_unicode +from rhodecode.lib.indexers import SCHEMA, IDX_NAME from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \ NodeDoesNotExistError @@ -117,10 +117,9 @@ class WhooshIndexingDaemon(object): """ node = self.get_node(repo, path) - + indexed = indexed_w_content = 0 # we just index the content of chosen files, and skip binary files if node.extension in INDEX_EXTENSIONS and not node.is_binary: - u_content = node.content if not isinstance(u_content, unicode): log.warning(' >> %s Could not get this content as unicode ' @@ -128,11 +127,13 @@ class WhooshIndexingDaemon(object): u_content = u'' else: log.debug(' >> %s [WITH CONTENT]' % path) + indexed_w_content += 1 else: log.debug(' >> %s' % path) # just index file name without it's content u_content = u'' + indexed += 1 writer.add_document( owner=unicode(repo.contact), @@ -142,6 +143,7 @@ class WhooshIndexingDaemon(object): modtime=self.get_node_mtime(node), extension=node.extension ) + return indexed, indexed_w_content def build_index(self): if os.path.exists(self.index_location): @@ -153,19 +155,25 @@ class WhooshIndexingDaemon(object): idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME) writer = idx.writer() - + log.debug('BUILDIN INDEX FOR EXTENSIONS %s' % INDEX_EXTENSIONS) for repo_name, repo in self.repo_paths.items(): log.debug('building index @ %s' % repo.path) - + i_cnt = iwc_cnt = 0 for idx_path in self.get_paths(repo): - self.add_doc(writer, idx_path, repo, repo_name) + i, iwc = self.add_doc(writer, idx_path, repo, repo_name) + i_cnt += i + iwc_cnt += iwc + log.debug('added %s files %s with content for repo %s' % ( + i_cnt + iwc_cnt, iwc_cnt, repo.path) + ) log.debug('>> COMMITING CHANGES <<') writer.commit(merge=True) log.debug('>>> FINISHED BUILDING INDEX <<<') def update_index(self): - log.debug('STARTING INCREMENTAL INDEXING UPDATE') + log.debug('STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s' % + INDEX_EXTENSIONS) idx = open_dir(self.index_location, indexname=self.indexname) # The set of all paths in the index @@ -204,14 +212,19 @@ class WhooshIndexingDaemon(object): # Loop over the files in the filesystem # Assume we have a function that gathers the filenames of the # documents to be indexed + ri_cnt = riwc_cnt = 0 for repo_name, repo in self.repo_paths.items(): for path in self.get_paths(repo): if path in to_index or path not in indexed_paths: # This is either a file that's changed, or a new file # that wasn't indexed before. So index it! - self.add_doc(writer, path, repo, repo_name) + i, iwc = self.add_doc(writer, path, repo, repo_name) log.debug('re indexing %s' % path) - + ri_cnt += i + riwc_cnt += iwc + log.debug('added %s files %s with content for repo %s' % ( + ri_cnt + riwc_cnt, riwc_cnt, repo.path) + ) log.debug('>> COMMITING CHANGES <<') writer.commit(merge=True) log.debug('>>> FINISHED REBUILDING INDEX <<<') diff --git a/rhodecode/lib/markup_renderer.py b/rhodecode/lib/markup_renderer.py index f99ca093..8f17f52f 100644 --- a/rhodecode/lib/markup_renderer.py +++ b/rhodecode/lib/markup_renderer.py @@ -27,7 +27,7 @@ import re import logging -from rhodecode.lib import safe_unicode +from rhodecode.lib.utils2 import safe_unicode log = logging.getLogger(__name__) diff --git a/rhodecode/lib/middleware/https_fixup.py b/rhodecode/lib/middleware/https_fixup.py index 4ecf95c1..5e1bab03 100644 --- a/rhodecode/lib/middleware/https_fixup.py +++ b/rhodecode/lib/middleware/https_fixup.py @@ -23,7 +23,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from rhodecode.lib import str2bool +from rhodecode.lib.utils2 import str2bool class HttpsFixup(object): diff --git a/rhodecode/lib/middleware/simplegit.py b/rhodecode/lib/middleware/simplegit.py index 591d408c..98df8e65 100644 --- a/rhodecode/lib/middleware/simplegit.py +++ b/rhodecode/lib/middleware/simplegit.py @@ -69,7 +69,7 @@ from dulwich.web import HTTPGitApplication from paste.httpheaders import REMOTE_USER, AUTH_TYPE -from rhodecode.lib import safe_str +from rhodecode.lib.utils2 import safe_str from rhodecode.lib.base import BaseVCSController from rhodecode.lib.auth import get_container_username from rhodecode.lib.utils import is_valid_repo diff --git a/rhodecode/lib/middleware/simplehg.py b/rhodecode/lib/middleware/simplehg.py index d7f4a869..9ccbbe51 100644 --- a/rhodecode/lib/middleware/simplehg.py +++ b/rhodecode/lib/middleware/simplehg.py @@ -34,7 +34,7 @@ from mercurial.hgweb import hgweb_mod from paste.httpheaders import REMOTE_USER, AUTH_TYPE -from rhodecode.lib import safe_str +from rhodecode.lib.utils2 import safe_str from rhodecode.lib.base import BaseVCSController from rhodecode.lib.auth import get_container_username from rhodecode.lib.utils import make_ui, is_valid_repo, ui_sections diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py index 83a7b87b..4ab41ac2 100644 --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -54,7 +54,8 @@ from rhodecode.model.db import Repository, User, RhodeCodeUi, \ UserLog, RepoGroup, RhodeCodeSetting, UserRepoGroupToPerm from rhodecode.model.meta import Session from rhodecode.model.repos_group import ReposGroupModel -from rhodecode.lib import safe_str, safe_unicode +from rhodecode.lib.utils2 import safe_str, safe_unicode +from rhodecode.lib.vcs.utils.fakemod import create_module log = logging.getLogger(__name__) @@ -62,7 +63,8 @@ REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*') def recursive_replace(str_, replace=' '): - """Recursive replace of given sign to just one instance + """ + Recursive replace of given sign to just one instance :param str_: given string :param replace: char to find and replace multiple instances @@ -80,7 +82,8 @@ def recursive_replace(str_, replace=' '): def repo_name_slug(value): - """Return slug of name of repository + """ + Return slug of name of repository This function is called on each creation/modification of repository to prevent bad names in repo """ @@ -263,7 +266,8 @@ ui_sections = ['alias', 'auth', def make_ui(read_from='file', path=None, checkpaths=True): - """A function that will read python rc files or database + """ + A function that will read python rc files or database and make an mercurial ui object from read options :param path: path to mercurial config file @@ -489,6 +493,30 @@ def add_cache(settings): beaker.cache.cache_regions[region] = region_settings +def load_rcextensions(root_path): + import rhodecode + from rhodecode.config import conf + + path = os.path.join(root_path, 'rcextensions', '__init__.py') + if os.path.isfile(path): + rcext = create_module('rc', path) + EXT = rhodecode.EXTENSIONS = rcext + log.debug('Found rcextensions now loading %s...' % rcext) + + # Additional mappings that are not present in the pygments lexers + conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {})) + + #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present) + + if getattr(EXT, 'INDEX_EXTENSIONS', []) != []: + log.debug('settings custom INDEX_EXTENSIONS') + conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', []) + + #ADDITIONAL MAPPINGS + log.debug('adding extra into INDEX_EXTENSIONS') + conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', [])) + + #============================================================================== # TEST FUNCTIONS AND CREATORS #============================================================================== diff --git a/rhodecode/lib/utils2.py b/rhodecode/lib/utils2.py new file mode 100644 index 00000000..04f4bee9 --- /dev/null +++ b/rhodecode/lib/utils2.py @@ -0,0 +1,405 @@ +# -*- coding: utf-8 -*- +""" + rhodecode.lib.utils + ~~~~~~~~~~~~~~~~~~~ + + Some simple helper functions + + :created_on: Jan 5, 2011 + :author: marcink + :copyright: (C) 2011-2012 Marcin Kuzminski <marcin@python-works.com> + :license: GPLv3, see COPYING for more details. +""" +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import re +from rhodecode.lib.vcs.utils.lazy import LazyProperty + + +def __get_lem(): + """ + Get language extension map based on what's inside pygments lexers + """ + from pygments import lexers + from string import lower + from collections import defaultdict + + d = defaultdict(lambda: []) + + def __clean(s): + s = s.lstrip('*') + s = s.lstrip('.') + + if s.find('[') != -1: + exts = [] + start, stop = s.find('['), s.find(']') + + for suffix in s[start + 1:stop]: + exts.append(s[:s.find('[')] + suffix) + return map(lower, exts) + else: + return map(lower, [s]) + + for lx, t in sorted(lexers.LEXERS.items()): + m = map(__clean, t[-2]) + if m: + m = reduce(lambda x, y: x + y, m) + for ext in m: + desc = lx.replace('Lexer', '') + d[ext].append(desc) + + return dict(d) + +def str2bool(_str): + """ + returs True/False value from given string, it tries to translate the + string into boolean + + :param _str: string value to translate into boolean + :rtype: boolean + :returns: boolean from given string + """ + if _str is None: + return False + if _str in (True, False): + return _str + _str = str(_str).strip().lower() + return _str in ('t', 'true', 'y', 'yes', 'on', '1') + + +def convert_line_endings(line, mode): + """ + Converts a given line "line end" accordingly to given mode + + Available modes are:: + 0 - Unix + 1 - Mac + 2 - DOS + + :param line: given line to convert + :param mode: mode to convert to + :rtype: str + :return: converted line according to mode + """ + from string import replace + + if mode == 0: + line = replace(line, '\r\n', '\n') + line = replace(line, '\r', '\n') + elif mode == 1: + line = replace(line, '\r\n', '\r') + line = replace(line, '\n', '\r') + elif mode == 2: + line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line) + return line + + +def detect_mode(line, default): + """ + Detects line break for given line, if line break couldn't be found + given default value is returned + + :param line: str line + :param default: default + :rtype: int + :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS + """ + if line.endswith('\r\n'): + return 2 + elif line.endswith('\n'): + return 0 + elif line.endswith('\r'): + return 1 + else: + return default + + +def generate_api_key(username, salt=None): + """ + Generates unique API key for given username, if salt is not given + it'll be generated from some random string + + :param username: username as string + :param salt: salt to hash generate KEY + :rtype: str + :returns: sha1 hash from username+salt + """ + from tempfile import _RandomNameSequence + import hashlib + + if salt is None: + salt = _RandomNameSequence().next() + + return hashlib.sha1(username + salt).hexdigest() + + +def safe_unicode(str_, from_encoding=None): + """ + safe unicode function. Does few trick to turn str_ into unicode + + In case of UnicodeDecode error we try to return it with encoding detected + by chardet library if it fails fallback to unicode with errors replaced + + :param str_: string to decode + :rtype: unicode + :returns: unicode object + """ + if isinstance(str_, unicode): + return str_ + + if not from_encoding: + import rhodecode + DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8') + from_encoding = DEFAULT_ENCODING + + try: + return unicode(str_) + except UnicodeDecodeError: + pass + + try: + return unicode(str_, from_encoding) + except UnicodeDecodeError: + pass + + try: + import chardet + encoding = chardet.detect(str_)['encoding'] + if encoding is None: + raise Exception() + return str_.decode(encoding) + except (ImportError, UnicodeDecodeError, Exception): + return unicode(str_, from_encoding, 'replace') + + +def safe_str(unicode_, to_encoding=None): + """ + safe str function. Does few trick to turn unicode_ into string + + In case of UnicodeEncodeError we try to return it with encoding detected + by chardet library if it fails fallback to string with errors replaced + + :param unicode_: unicode to encode + :rtype: str + :returns: str object + """ + + # if it's not basestr cast to str + if not isinstance(unicode_, basestring): + return str(unicode_) + + if isinstance(unicode_, str): + return unicode_ + + if not to_encoding: + import rhodecode + DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8') + to_encoding = DEFAULT_ENCODING + + try: + return unicode_.encode(to_encoding) + except UnicodeEncodeError: + pass + + try: + import chardet + encoding = chardet.detect(unicode_)['encoding'] + print encoding + if encoding is None: + raise UnicodeEncodeError() + + return unicode_.encode(encoding) + except (ImportError, UnicodeEncodeError): + return unicode_.encode(to_encoding, 'replace') + + return safe_str + + +def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): + """ + Custom engine_from_config functions that makes sure we use NullPool for + file based sqlite databases. This prevents errors on sqlite. This only + applies to sqlalchemy versions < 0.7.0 + + """ + import sqlalchemy + from sqlalchemy import engine_from_config as efc + import logging + + if int(sqlalchemy.__version__.split('.')[1]) < 7: + + # This solution should work for sqlalchemy < 0.7.0, and should use + # proxy=TimerProxy() for execution time profiling + + from sqlalchemy.pool import NullPool + url = configuration[prefix + 'url'] + + if url.startswith('sqlite'): + kwargs.update({'poolclass': NullPool}) + return efc(configuration, prefix, **kwargs) + else: + import time + from sqlalchemy import event + from sqlalchemy.engine import Engine + + log = logging.getLogger('sqlalchemy.engine') + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38) + engine = efc(configuration, prefix, **kwargs) + + def color_sql(sql): + COLOR_SEQ = "\033[1;%dm" + COLOR_SQL = YELLOW + normal = '\x1b[0m' + return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal]) + + if configuration['debug']: + #attach events only for debug configuration + + def before_cursor_execute(conn, cursor, statement, + parameters, context, executemany): + context._query_start_time = time.time() + log.info(color_sql(">>>>> STARTING QUERY >>>>>")) + + + def after_cursor_execute(conn, cursor, statement, + parameters, context, executemany): + total = time.time() - context._query_start_time + log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total)) + + event.listen(engine, "before_cursor_execute", + before_cursor_execute) + event.listen(engine, "after_cursor_execute", + after_cursor_execute) + + return engine + + +def age(curdate): + """ + turns a datetime into an age string. + + :param curdate: datetime object + :rtype: unicode + :returns: unicode words describing age + """ + + from datetime import datetime + from webhelpers.date import time_ago_in_words + + _ = lambda s: s + + if not curdate: + return '' + + agescales = [(_(u"year"), 3600 * 24 * 365), + (_(u"month"), 3600 * 24 * 30), + (_(u"day"), 3600 * 24), + (_(u"hour"), 3600), + (_(u"minute"), 60), + (_(u"second"), 1), ] + + age = datetime.now() - curdate + age_seconds = (age.days * agescales[2][1]) + age.seconds + pos = 1 + for scale in agescales: + if scale[1] <= age_seconds: + if pos == 6: + pos = 5 + return '%s %s' % (time_ago_in_words(curdate, + agescales[pos][0]), _('ago')) + pos += 1 + + return _(u'just now') + + +def uri_filter(uri): + """ + Removes user:password from given url string + + :param uri: + :rtype: unicode + :returns: filtered list of strings + """ + if not uri: + return '' + + proto = '' + + for pat in ('https://', 'http://'): + if uri.startswith(pat): + uri = uri[len(pat):] + proto = pat + break + + # remove passwords and username + uri = uri[uri.find('@') + 1:] + + # get the port + cred_pos = uri.find(':') + if cred_pos == -1: + host, port = uri, None + else: + host, port = uri[:cred_pos], uri[cred_pos + 1:] + + return filter(None, [proto, host, port]) + + +def credentials_filter(uri): + """ + Returns a url with removed credentials + + :param uri: + """ + + uri = uri_filter(uri) + #check if we have port + if len(uri) > 2 and uri[2]: + uri[2] = ':' + uri[2] + + return ''.join(uri) + + +def get_changeset_safe(repo, rev): + """ + Safe version of get_changeset if this changeset doesn't exists for a + repo it returns a Dummy one instead + + :param repo: + :param rev: + """ + from rhodecode.lib.vcs.backends.base import BaseRepository + from rhodecode.lib.vcs.exceptions import RepositoryError + if not isinstance(repo, BaseRepository): + raise Exception('You must pass an Repository ' + 'object as first argument got %s', type(repo)) + + try: + cs = repo.get_changeset(rev) + except RepositoryError: + from rhodecode.lib.utils import EmptyChangeset + cs = EmptyChangeset(requested_revision=rev) + return cs + + +def extract_mentioned_users(s): + """ + Returns unique usernames from given string s that have @mention + + :param s: string to get mentions + """ + usrs = {} + for username in re.findall(r'(?:^@|\s@)(\w+)', s): + usrs[username] = username + + return sorted(usrs.keys()) diff --git a/rhodecode/model/comment.py b/rhodecode/model/comment.py index 3753198c..51d6bf1e 100644 --- a/rhodecode/model/comment.py +++ b/rhodecode/model/comment.py @@ -29,7 +29,7 @@ import traceback from pylons.i18n.translation import _ from sqlalchemy.util.compat import defaultdict -from rhodecode.lib import extract_mentioned_users +from rhodecode.lib.utils2 import extract_mentioned_users from rhodecode.lib import helpers as h from rhodecode.model import BaseModel from rhodecode.model.db import ChangesetComment, User, Repository, Notification diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py index 864c006b..1e3b8756 100755 --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -39,7 +39,8 @@ from rhodecode.lib.vcs.utils.helpers import get_scm from rhodecode.lib.vcs.exceptions import VCSError from rhodecode.lib.vcs.utils.lazy import LazyProperty -from rhodecode.lib import str2bool, safe_str, get_changeset_safe, safe_unicode +from rhodecode.lib.utils2 import str2bool, safe_str, get_changeset_safe, \ + safe_unicode from rhodecode.lib.compat import json from rhodecode.lib.caching_query import FromCache diff --git a/rhodecode/model/notification.py b/rhodecode/model/notification.py index c8ef0f80..d23805c4 100644 --- a/rhodecode/model/notification.py +++ b/rhodecode/model/notification.py @@ -32,6 +32,7 @@ import datetime from pylons.i18n.translation import _ import rhodecode +from rhodecode.config.conf import DATETIME_FORMAT from rhodecode.lib import helpers as h from rhodecode.model import BaseModel from rhodecode.model.db import Notification, User, UserNotification @@ -176,8 +177,6 @@ class NotificationModel(BaseModel): notification.TYPE_REGISTRATION: _('registered in RhodeCode') } - DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" - tmpl = "%(user)s %(action)s %(when)s" if show_age: when = h.age(notification.created_on) diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py index 90609bb5..700da381 100644 --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -30,8 +30,7 @@ from datetime import datetime from rhodecode.lib.vcs.backends import get_backend -from rhodecode.lib import LazyProperty -from rhodecode.lib import safe_str, safe_unicode +from rhodecode.lib.utils2 import LazyProperty, safe_str, safe_unicode from rhodecode.lib.caching_query import FromCache from rhodecode.lib.hooks import log_create_repository diff --git a/rhodecode/model/repos_group.py b/rhodecode/model/repos_group.py index 5bc435e7..8791b21b 100644 --- a/rhodecode/model/repos_group.py +++ b/rhodecode/model/repos_group.py @@ -28,7 +28,7 @@ import logging import traceback import shutil -from rhodecode.lib import LazyProperty +from rhodecode.lib.utils2 import LazyProperty from rhodecode.model import BaseModel from rhodecode.model.db import RepoGroup, RhodeCodeUi, UserRepoGroupToPerm, \ diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py index cfeb1fe7..9d2aa466 100644 --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -35,7 +35,7 @@ from rhodecode.lib.vcs.nodes import FileNode from rhodecode import BACKENDS from rhodecode.lib import helpers as h -from rhodecode.lib import safe_str +from rhodecode.lib.utils2 import safe_str from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, \ action_logger, EmptyChangeset, REMOVED_REPO_PAT diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py index 8bb0f3b8..d2411c27 100644 --- a/rhodecode/model/user.py +++ b/rhodecode/model/user.py @@ -29,7 +29,7 @@ import traceback from pylons import url from pylons.i18n.translation import _ -from rhodecode.lib import safe_unicode +from rhodecode.lib.utils2 import safe_unicode, generate_api_key from rhodecode.lib.caching_query import FromCache from rhodecode.model import BaseModel @@ -40,7 +40,7 @@ from rhodecode.lib.exceptions import DefaultUserException, \ UserOwnsReposException from sqlalchemy.exc import DatabaseError -from rhodecode.lib import generate_api_key + from sqlalchemy.orm import joinedload log = logging.getLogger(__name__) diff --git a/rhodecode/public/css/style.css b/rhodecode/public/css/style.css index 54707914..52dab6bd 100644 --- a/rhodecode/public/css/style.css +++ b/rhodecode/public/css/style.css @@ -3112,7 +3112,12 @@ table.code-browser .browser-dir { top: 5px;
width: 16px;
}
-
+div#legend_data{
+ padding-left:10px;
+}
+div#legend_container table{
+ border: none !important;
+}
div#legend_container table,div#legend_choices table {
width: auto !important;
}
diff --git a/rhodecode/tests/functional/test_login.py b/rhodecode/tests/functional/test_login.py index deeca967..0791cdc4 100644 --- a/rhodecode/tests/functional/test_login.py +++ b/rhodecode/tests/functional/test_login.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- from rhodecode.tests import * from rhodecode.model.db import User, Notification -from rhodecode.lib import generate_api_key +from rhodecode.lib.utils2 import generate_api_key from rhodecode.lib.auth import check_password from rhodecode.model.meta import Session + class TestLoginController(TestController): def tearDown(self): diff --git a/rhodecode/tests/test_libs.py b/rhodecode/tests/test_libs.py index c01fac73..33fdd7fb 100644 --- a/rhodecode/tests/test_libs.py +++ b/rhodecode/tests/test_libs.py @@ -65,22 +65,20 @@ TEST_URLS += [ class TestLibs(unittest.TestCase): - def test_uri_filter(self): - from rhodecode.lib import uri_filter + from rhodecode.lib.utils2 import uri_filter for url in TEST_URLS: self.assertEqual(uri_filter(url[0]), url[1]) def test_credentials_filter(self): - from rhodecode.lib import credentials_filter + from rhodecode.lib.utils2 import credentials_filter for url in TEST_URLS: self.assertEqual(credentials_filter(url[0]), url[2]) - def test_str2bool(self): - from rhodecode.lib import str2bool + from rhodecode.lib.utils2 import str2bool test_cases = [ ('t', True), ('true', True), @@ -103,9 +101,8 @@ class TestLibs(unittest.TestCase): for case in test_cases: self.assertEqual(str2bool(case[0]), case[1]) - def test_mention_extractor(self): - from rhodecode.lib import extract_mentioned_users + from rhodecode.lib.utils2 import extract_mentioned_users sample = ("@first hi there @marcink here's my email marcin@email.com " "@lukaszb check it pls @ ttwelve @D[] @one@two@three " "@MARCIN @maRCiN @2one_more22") |