fixes #200, rewrote the whole caching mechanism to get rid of such problems. Now cached instances are attached

to db repository instance, and then fetched from cache. Also made all current test work.

--HG--
branch : beta

21 files changed, 398 insertions, 238 deletions

diff --git a/rhodecode/controllers/admin/repos.py b/rhodecode/controllers/admin/repos.py
index 59c59c63..9832637b 100644
--- a/rhodecode/controllers/admin/repos.py
+++ b/rhodecode/controllers/admin/repos.py
@@ -89,10 +89,8 @@ class ReposController(BaseController):
         """
         self.__load_defaults()
 
-        repo, dbrepo = ScmModel().get(repo_name, retval='repo')
-
-        repo_model = RepoModel()
-        c.repo_info = repo_model.get_by_repo_name(repo_name)
+        c.repo_info = db_repo = Repository.by_repo_name(repo_name)
+        repo = scm_repo = db_repo.scm_instance
 
         if c.repo_info is None:
             h.flash(_('%s repository is not mapped to db perhaps'
@@ -153,10 +151,9 @@ class ReposController(BaseController):
         """GET /repos: All items in the collection"""
         # url('repos')
 
-        all_repos = [r.repo_name for r in Repository.query().all()]
-
-        cached_repo_list = ScmModel().get_repos(all_repos)
-        c.repos_list = sorted(cached_repo_list, key=itemgetter('name_sort'))
+        c.repos_list = ScmModel().get_repos(Repository.query()
+                                            .order_by(Repository.repo_name)
+                                            .all(), sort_key='name_sort')
         return render('admin/repos/repos.html')
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository')
diff --git a/rhodecode/controllers/admin/repos_groups.py b/rhodecode/controllers/admin/repos_groups.py
index 4fff350f..323e9e07 100644
--- a/rhodecode/controllers/admin/repos_groups.py
+++ b/rhodecode/controllers/admin/repos_groups.py
@@ -181,12 +181,14 @@ class ReposGroupsController(BaseController):
         """GET /repos_groups/id: Show a specific item"""
         # url('repos_group', id=ID)
 
-        c.group = Group.get(id)
+        gr = c.group = Group.get(id)
+
         if c.group:
             c.group_repos = c.group.repositories.all()
         else:
             return redirect(url('repos_group'))
 
+
         sortables = ['name', 'description', 'last_change', 'tip', 'owner']
         current_sort = request.GET.get('sort', 'name')
         current_sort_slug = current_sort.replace('-', '')
@@ -201,18 +203,12 @@ class ReposGroupsController(BaseController):
         sort_key = current_sort_slug + '_sort'
 
         #overwrite our cached list with current filter
-        gr_filter = [r.repo_name for r in c.group_repos]
+        gr_filter = c.group_repos
         c.cached_repo_list = self.scm_model.get_repos(all_repos=gr_filter)
 
-        if c.sort_by.startswith('-'):
-            c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key),
-                                  reverse=True)
-        else:
-            c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key),
-                                  reverse=False)
-
-        c.repo_cnt = len(c.repos_list)
+        c.repos_list = c.cached_repo_list
 
+        c.repo_cnt = 0
 
         c.groups = self.sa.query(Group).order_by(Group.group_name)\
             .filter(Group.group_parent_id == id).all()
diff --git a/rhodecode/controllers/admin/settings.py b/rhodecode/controllers/admin/settings.py
index 743ea03b..62597965 100644
--- a/rhodecode/controllers/admin/settings.py
+++ b/rhodecode/controllers/admin/settings.py
@@ -258,9 +258,10 @@ class SettingsController(BaseController):
         # url('admin_settings_my_account')
 
         c.user = UserModel().get(self.rhodecode_user.user_id, cache=False)
-        all_repos = [r.repo_name for r in self.sa.query(Repository)\
+        all_repos = self.sa.query(Repository)\
                      .filter(Repository.user_id == c.user.user_id)\
-                     .order_by(func.lower(Repository.repo_name)).all()]
+                     .order_by(func.lower(Repository.repo_name)).all()
+
         c.user_repos = ScmModel().get_repos(all_repos)
 
         if c.user.username == 'default':
diff --git a/rhodecode/controllers/home.py b/rhodecode/controllers/home.py
index 7ba7c462..e6517b53 100644
--- a/rhodecode/controllers/home.py
+++ b/rhodecode/controllers/home.py
@@ -31,7 +31,7 @@ from paste.httpexceptions import HTTPBadRequest
 
 from rhodecode.lib.auth import LoginRequired
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model.db import Group
+from rhodecode.model.db import Group, Repository
 
 log = logging.getLogger(__name__)
 
@@ -56,15 +56,10 @@ class HomeController(BaseController):
 
         sort_key = current_sort_slug + '_sort'
 
-        if c.sort_by.startswith('-'):
-            c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key),
-                                  reverse=True)
-        else:
-            c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key),
-                                  reverse=False)
 
-        c.repo_cnt = len(c.repos_list)
+        c.repos_list = self.scm_model.get_repos(sort_key=sort_key)
 
+        c.repo_cnt = len(c.repos_list)
 
         c.groups = Group.query().filter(Group.group_parent_id == None).all()
 
@@ -73,8 +68,9 @@ class HomeController(BaseController):
 
     def repo_switcher(self):
         if request.is_xhr:
-            c.repos_list = sorted(c.cached_repo_list,
-                                  key=itemgetter('name_sort'), reverse=False)
+            all_repos = Repository.query().order_by(Repository.repo_name).all()
+            c.repos_list = self.scm_model.get_repos(all_repos,
+                                                    sort_key='name_sort')
             return render('/repo_switcher_list.html')
         else:
             return HTTPBadRequest()
diff --git a/rhodecode/controllers/settings.py b/rhodecode/controllers/settings.py
index 3fc6abb6..d7e7a6b8 100644
--- a/rhodecode/controllers/settings.py
+++ b/rhodecode/controllers/settings.py
@@ -155,6 +155,7 @@ class SettingsController(BaseRepoController):
             invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('deleted repository %s') % repo_name, category='success')
         except Exception:
+            log.error(traceback.format_exc())
             h.flash(_('An error occurred during deletion of %s') % repo_name,
                     category='error')
 
@@ -205,4 +206,9 @@ class SettingsController(BaseRepoController):
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8")
+        except Exception:
+            log.error(traceback.format_exc())
+            h.flash(_('An error occurred during repository forking %s') %
+                    repo_name, category='error')
+
         return redirect(url('home'))
diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py
index 0d5bbaa9..3bf6b6dd 100644
--- a/rhodecode/lib/base.py
+++ b/rhodecode/lib/base.py
@@ -12,6 +12,7 @@ from rhodecode.lib.utils import get_repo_slug
 from rhodecode.model import meta
 from rhodecode.model.scm import ScmModel
 from rhodecode import BACKENDS
+from rhodecode.model.db import Repository
 
 
 class BaseController(WSGIController):
@@ -26,7 +27,7 @@ class BaseController(WSGIController):
 
         self.sa = meta.Session()
         self.scm_model = ScmModel(self.sa)
-        c.cached_repo_list = self.scm_model.get_repos()
+
         #c.unread_journal = scm_model.get_unread_journal()
 
     def __call__(self, environ, start_response):
@@ -62,8 +63,7 @@ class BaseRepoController(BaseController):
         super(BaseRepoController, self).__before__()
         if c.repo_name:
 
-            c.rhodecode_repo, dbrepo = self.scm_model.get(c.repo_name,
-                                                          retval='repo')
+            c.rhodecode_repo = Repository.by_repo_name(c.repo_name).scm_instance
 
             if c.rhodecode_repo is not None:
                 c.repository_followers = \
diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py
index d79bd81d..868b4243 100644
--- a/rhodecode/lib/celerylib/tasks.py
+++ b/rhodecode/lib/celerylib/tasks.py
@@ -102,7 +102,6 @@ def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
                             ts_max_y)
     lockkey_path = dn(dn(dn(dn(os.path.abspath(__file__)))))
-    print jn(lockkey_path, lockkey)
     log.info('running task with lockkey %s', lockkey)
     try:
         lock = l = DaemonLock(jn(lockkey_path, lockkey))
diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py
index d69b257f..e8f7e644 100644
--- a/rhodecode/lib/helpers.py
+++ b/rhodecode/lib/helpers.py
@@ -372,8 +372,7 @@ def action_parser(user_log, feed=False):
         repo_name = user_log.repository.repo_name
 
         from rhodecode.model.scm import ScmModel
-        repo, dbrepo = ScmModel().get(repo_name, retval='repo',
-                                      invalidation_list=[])
+        repo = user_log.repository.scm_instance
 
         message = lambda rev: get_changeset_safe(repo, rev).message
         cs_links = []
diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py
index 1db0080c..28b8107e 100644
--- a/rhodecode/lib/utils.py
+++ b/rhodecode/lib/utils.py
@@ -472,7 +472,7 @@ def create_test_index(repo_location, full_index):
         shutil.rmtree(index_location)
 
     try:
-        l = DaemonLock(file=jn(dn(dn(index_location)), 'make_index.lock'))
+        l = DaemonLock(file=jn(dn(index_location), 'make_index.lock'))
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location)\
             .run(full_index=full_index)
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index c253b879..6421a072 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -26,13 +26,23 @@
 import os
 import logging
 import datetime
+import traceback
 from datetime import date
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
-from sqlalchemy.orm import relationship, backref
+from sqlalchemy.orm import relationship, backref, joinedload
 from sqlalchemy.orm.interfaces import MapperExtension
 
+from beaker.cache import cache_region, region_invalidate
+
+
+from vcs import get_backend
+from vcs.utils.helpers import get_scm
+from vcs.exceptions import RepositoryError, VCSError
+from vcs.utils.lazy import LazyProperty
+from vcs.nodes import FileNode
+
 from rhodecode.lib import str2bool
 from rhodecode.model.meta import Base, Session
 from rhodecode.model.caching_query import FromCache
@@ -150,6 +160,7 @@ class User(Base):
         return self.admin
 
     def __repr__(self):
+        return 'ahmmm'
         return "<%s('id:%s:%s')>" % (self.__class__.__name__,
                                      self.user_id, self.username)
 
@@ -266,8 +277,13 @@ class Repository(Base):
 
     @classmethod
     def by_repo_name(cls, repo_name):
-        return Session.query(cls).filter(cls.repo_name == repo_name).one()
+        q = Session.query(cls).filter(cls.repo_name == repo_name)
+
+        q = q.options(joinedload(Repository.fork))\
+            .options(joinedload(Repository.user))\
+            .options(joinedload(Repository.group))\
 
+        return q.one()
 
     @classmethod
     def get_repo_forks(cls, repo_id):
@@ -298,6 +314,127 @@ class Repository(Base):
     def groups_and_repo(self):
         return self.groups_with_parents, self.just_name
 
+    @LazyProperty
+    def repo_path(self):
+        """
+        Returns base full path for that repository means where it actually
+        exists on a filesystem
+        """
+
+        q = Session.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
+        return q.ui_value
+
+    @property
+    def repo_full_path(self):
+        p = [self.repo_path]
+        # we need to split the name by / since this is how we store the
+        # names in the database, but that eventually needs to be converted
+        # into a valid system path
+        p += self.repo_name.split('/')
+        return os.path.join(*p)
+
+    @property
+    def _ui(self):
+        """
+        Creates an db based ui object for this repository
+        """
+        from mercurial import ui
+        from mercurial import config
+        baseui = ui.ui()
+
+        #clean the baseui object
+        baseui._ocfg = config.config()
+        baseui._ucfg = config.config()
+        baseui._tcfg = config.config()
+
+
+        ret = Session.query(RhodeCodeUi)\
+            .options(FromCache("sql_cache_short",
+                               "repository_repo_ui")).all()
+
+        hg_ui = ret
+        for ui_ in hg_ui:
+            if ui_.ui_active:
+                log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,
+                          ui_.ui_key, ui_.ui_value)
+                baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
+
+        return baseui
+
+    #==========================================================================
+    # SCM CACHE INSTANCE
+    #==========================================================================
+
+    @property
+    def invalidate(self):
+        """
+        Returns Invalidation object if this repo should be invalidated
+        None otherwise. `cache_active = False` means that this cache
+        state is not valid and needs to be invalidated
+        """
+        return Session.query(CacheInvalidation)\
+            .filter(CacheInvalidation.cache_key == self.repo_name)\
+            .filter(CacheInvalidation.cache_active == False)\
+            .scalar()
+
+    @property
+    def set_invalidate(self):
+        """
+        set a cache for invalidation for this instance
+        """
+        inv = Session.query(CacheInvalidation)\
+            .filter(CacheInvalidation.cache_key == self.repo_name)\
+            .scalar()
+
+        if inv is None:
+            inv = CacheInvalidation(self.repo_name)
+        inv.cache_active = True
+        Session.add(inv)
+        Session.commit()
+
+    @property
+    def scm_instance(self):
+        return self.__get_instance(self.repo_name)
+
+    @property
+    def scm_instance_cached(self):
+        @cache_region('long_term')
+        def _c(repo_name):
+            return self.__get_instance(repo_name)
+
+        inv = self.invalidate
+        if inv:
+            region_invalidate(_c, None, self.repo_name)
+            #update our cache
+            inv.cache_key.cache_active = True
+            Session.add(inv)
+            Session.commit()
+
+        return _c(self.repo_name)
+
+    def __get_instance(self, repo_name):
+        try:
+            alias = get_scm(self.repo_full_path)[0]
+            log.debug('Creating instance of %s repository', alias)
+            backend = get_backend(alias)
+        except VCSError:
+            log.error(traceback.format_exc())
+            log.error('Perhaps this repository is in db and not in '
+                      'filesystem run rescan repositories with '
+                      '"destroy old data " option from admin panel')
+            return
+
+        if alias == 'hg':
+            repo = backend(self.repo_full_path, create=False,
+                           baseui=self._ui)
+            #skip hidden web repository
+            if repo._get_hidden():
+                return
+        else:
+            repo = backend(self.repo_full_path, create=False)
+
+        return repo
+
 
 class Group(Base):
     __tablename__ = 'groups'
diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py
index 57ccaa05..f7cfc9b3 100644
--- a/rhodecode/model/forms.py
+++ b/rhodecode/model/forms.py
@@ -280,6 +280,13 @@ def ValidRepoName(edit, old_data):
 
     return _ValidRepoName
 
+def ValidForkName():
+    class _ValidForkName(formencode.validators.FancyValidator):
+        def to_python(self, value, state):
+            return value
+    return _ValidForkName
+
+
 def SlugifyName():
     class _SlugifyName(formencode.validators.FancyValidator):
 
@@ -326,6 +333,7 @@ def ValidForkType(old_data):
             if old_data['repo_type'] != value:
                 raise formencode.Invalid(_('Fork have to be the same '
                                            'type as original'), value, state)
+
             return value
     return _ValidForkType
 
@@ -583,6 +591,9 @@ def RepoForkForm(edit=False, old_data={}, supported_backends=BACKENDS.keys()):
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
         repo_type = All(ValidForkType(old_data), OneOf(supported_backends))
+
+        chained_validators = [ValidForkName()]
+
     return _RepoForkForm
 
 def RepoSettingsForm(edit=False, old_data={}):
diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py
index 384aea74..007941e6 100644
--- a/rhodecode/model/repo.py
+++ b/rhodecode/model/repo.py
@@ -70,28 +70,6 @@ class RepoModel(BaseModel):
                                           "get_repo_%s" % repo_name))
         return repo.scalar()
 
-    def get_full(self, repo_name, cache=False, invalidate=False):
-        repo = self.sa.query(Repository)\
-            .options(joinedload(Repository.fork))\
-            .options(joinedload(Repository.user))\
-            .options(joinedload(Repository.group))\
-            .filter(Repository.repo_name == repo_name)\
-
-        if cache:
-            repo = repo.options(FromCache("sql_cache_long",
-                                          "get_repo_full_%s" % repo_name))
-        if invalidate and cache:
-            repo.invalidate()
-
-        ret = repo.scalar()
-
-        #make transient for sake of errors
-        make_transient(ret)
-        for k in ['fork', 'user', 'group']:
-            attr = getattr(ret, k, False)
-            if attr:
-                make_transient(attr)
-        return ret
 
     def get_users_js(self):
 
@@ -193,12 +171,13 @@ class RepoModel(BaseModel):
             raise
 
     def create(self, form_data, cur_user, just_db=False, fork=False):
+
         try:
             if fork:
                 #force str since hg doesn't go with unicode
                 repo_name = str(form_data['fork_name'])
                 org_name = str(form_data['repo_name'])
-                org_full_name = str(form_data['repo_name_full'])
+                org_full_name = org_name#str(form_data['fork_name_full'])
 
             else:
                 org_name = repo_name = str(form_data['repo_name'])
@@ -208,7 +187,10 @@ class RepoModel(BaseModel):
             new_repo.enable_statistics = False
             for k, v in form_data.items():
                 if k == 'repo_name':
-                    v = repo_name_full
+                    if fork:
+                        v = repo_name
+                    else:
+                        v = repo_name_full
                 if k == 'repo_group':
                     k = 'group_id'
 
@@ -216,7 +198,7 @@ class RepoModel(BaseModel):
 
             if fork:
                 parent_repo = self.sa.query(Repository)\
-                        .filter(Repository.repo_name == org_full_name).scalar()
+                        .filter(Repository.repo_name == org_full_name).one()
                 new_repo.fork = parent_repo
 
             new_repo.user_id = cur_user.user_id
diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py
index ae151923..097c0f60 100644
--- a/rhodecode/model/scm.py
+++ b/rhodecode/model/scm.py
@@ -70,6 +70,75 @@ class RepoTemp(object):
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 
+class CachedRepoList(object):
+
+    def __init__(self, db_repo_list, invalidation_list, repos_path,
+                 order_by=None):
+        self.db_repo_list = db_repo_list
+        self.invalidation_list = invalidation_list
+        self.repos_path = repos_path
+        self.order_by = order_by
+        self.reversed = (order_by or '').startswith('-')
+
+    def __len__(self):
+        return len(self.db_repo_list)
+
+    def __repr__(self):
+        return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
+
+    def __iter__(self):
+        for db_repo in self.db_repo_list:
+            dbr = db_repo
+
+            # invalidate the repo cache if needed before getting the 
+            # scm instance
+
+            scm_invalidate = False
+            if self.invalidation_list is not None:
+                scm_invalidate = dbr.repo_name in self.invalidation_list
+
+            if scm_invalidate:
+                log.info('invalidating cache for repository %s',
+                         dbr.repo_name)
+                db_repo.set_invalidate
+
+            scmr = db_repo.scm_instance_cached
+
+            #check permission at this level
+            if not HasRepoPermissionAny('repository.read',
+                                        'repository.write',
+                                        'repository.admin')(dbr.repo_name,
+                                                            'get repo check'):
+                continue
+
+
+
+
+
+            last_change = scmr.last_change
+            tip = h.get_changeset_safe(scmr, 'tip')
+
+            tmp_d = {}
+            tmp_d['name'] = dbr.repo_name
+            tmp_d['name_sort'] = tmp_d['name'].lower()
+            tmp_d['description'] = dbr.description
+            tmp_d['description_sort'] = tmp_d['description']
+            tmp_d['last_change'] = last_change
+            tmp_d['last_change_sort'] = time.mktime(last_change \
+                                                    .timetuple())
+            tmp_d['tip'] = tip.raw_id
+            tmp_d['tip_sort'] = tip.revision
+            tmp_d['rev'] = tip.revision
+            tmp_d['contact'] = dbr.user.full_contact
+            tmp_d['contact_sort'] = tmp_d['contact']
+            tmp_d['owner_sort'] = tmp_d['contact']
+            tmp_d['repo_archives'] = list(scmr._get_archives())
+            tmp_d['last_msg'] = tip.message
+            tmp_d['repo'] = scmr
+            tmp_d['dbrepo'] = dbr.get_dict()
+            tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork \
+                                                                    else {}
+            yield tmp_d
 
 class ScmModel(BaseModel):
     """Generic Scm Model
@@ -118,7 +187,7 @@ class ScmModel(BaseModel):
 
         return repos_list
 
-    def get_repos(self, all_repos=None):
+    def get_repos(self, all_repos=None, sort_key=None):
         """
         Get all repos from db and for each repo create it's
         backend instance and fill that backed with information from database
@@ -127,120 +196,21 @@ class ScmModel(BaseModel):
             give specific repositories list, good for filtering
         """
         if all_repos is None:
-            repos = self.sa.query(Repository)\
+            all_repos = self.sa.query(Repository)\
                         .filter(Repository.group_id == None)\
                         .order_by(Repository.repo_name).all()
-            all_repos = [r.repo_name for r in repos]
 
         #get the repositories that should be invalidated
         invalidation_list = [str(x.cache_key) for x in \
                              self.sa.query(CacheInvalidation.cache_key)\
                              .filter(CacheInvalidation.cache_active == False)\
                              .all()]
-        for r_name in all_repos:
-            r_dbr = self.get(r_name, invalidation_list)
-            if r_dbr is not None:
-                repo, dbrepo = r_dbr
-
-                if repo is None or dbrepo is None:
-                    log.error('Repository "%s" looks somehow corrupted '
-                              'fs-repo:%s,db-repo:%s both values should be '
-                              'present', r_name, repo, dbrepo)
-                    continue
-                last_change = repo.last_change
-                tip = h.get_changeset_safe(repo, 'tip')
-
-                tmp_d = {}
-                tmp_d['name'] = dbrepo.repo_name
-                tmp_d['name_sort'] = tmp_d['name'].lower()
-                tmp_d['description'] = dbrepo.description
-                tmp_d['description_sort'] = tmp_d['description']
-                tmp_d['last_change'] = last_change
-                tmp_d['last_change_sort'] = time.mktime(last_change \
-                                                        .timetuple())
-                tmp_d['tip'] = tip.raw_id
-                tmp_d['tip_sort'] = tip.revision
-                tmp_d['rev'] = tip.revision
-                tmp_d['contact'] = dbrepo.user.full_contact
-                tmp_d['contact_sort'] = tmp_d['contact']
-                tmp_d['owner_sort'] = tmp_d['contact']
-                tmp_d['repo_archives'] = list(repo._get_archives())
-                tmp_d['last_msg'] = tip.message
-                tmp_d['repo'] = repo
-                tmp_d['dbrepo'] = dbrepo.get_dict()
-                tmp_d['dbrepo_fork'] = dbrepo.fork.get_dict() if dbrepo.fork \
-                                                                        else {}
-                yield tmp_d
-
-    def get(self, repo_name, invalidation_list=None, retval='all'):
-        """Returns a tuple of Repository,DbRepository,
-        Get's repository from given name, creates BackendInstance and
-        propagates it's data from database with all additional information
-
-        :param repo_name:
-        :param invalidation_list: if a invalidation list is given the get
-            method should not manually check if this repository needs
-            invalidation and just invalidate the repositories in list
-        :param retval: string specifing what to return one of 'repo','dbrepo',
-            'all'if repo or dbrepo is given it'll just lazy load chosen type
-            and return None as the second
-        """
-        if not HasRepoPermissionAny('repository.read', 'repository.write',
-                            'repository.admin')(repo_name, 'get repo check'):
-            return
-
-        #======================================================================
-        # CACHE FUNCTION
-        #======================================================================
-        @cache_region('long_term')
-        def _get_repo(repo_name):
-
-            repo_path = os.path.join(self.repos_path, repo_name)
 
-            try:
-                alias = get_scm(repo_path)[0]
-                log.debug('Creating instance of %s repository', alias)
-                backend = get_backend(alias)
-            except VCSError:
-                log.error(traceback.format_exc())
-                log.error('Perhaps this repository is in db and not in '
-                          'filesystem run rescan repositories with '
-                          '"destroy old data " option from admin panel')
-                return
+        repo_iter = CachedRepoList(all_repos, invalidation_list,
+                                   repos_path=self.repos_path,
+                                   order_by=sort_key)
 
-            if alias == 'hg':
-                repo = backend(repo_path, create=False, baseui=make_ui('db'))
-                #skip hidden web repository
-                if repo._get_hidden():
-                    return
-            else:
-                repo = backend(repo_path, create=False)
-
-            return repo
-
-        pre_invalidate = True
-        dbinvalidate = False
-
-        if invalidation_list is not None:
-            pre_invalidate = repo_name in invalidation_list
-
-        if pre_invalidate:
-            #this returns object to invalidate
-            invalidate = self._should_invalidate(repo_name)
-            if invalidate:
-                log.info('invalidating cache for repository %s', repo_name)
-                region_invalidate(_get_repo, None, repo_name)
-                self._mark_invalidated(invalidate)
-                dbinvalidate = True
-
-        r, dbr = None, None
-        if retval == 'repo' or 'all':
-            r = _get_repo(repo_name)
-        if retval == 'dbrepo' or 'all':
-            dbr = RepoModel().get_full(repo_name, cache=True,
-                                          invalidate=dbinvalidate)
-
-        return r, dbr
+        return repo_iter
 
     def mark_for_invalidation(self, repo_name):
         """Puts cache invalidation task into db for
diff --git a/rhodecode/templates/admin/users/user_edit.html b/rhodecode/templates/admin/users/user_edit.html
index 760d8a12..7fb8b815 100644
--- a/rhodecode/templates/admin/users/user_edit.html
+++ b/rhodecode/templates/admin/users/user_edit.html
@@ -65,7 +65,7 @@
                     <label for="new_password">${_('New password')}:</label>
                 </div>
                 <div class="input">
-                    ${h.password('new_password',class_='medium')}
+                    ${h.password('new_password',class_='medium',autocomplete="off")}
                 </div>
              </div>
             
diff --git a/rhodecode/templates/admin/users/user_edit_my_account.html b/rhodecode/templates/admin/users/user_edit_my_account.html
index e7075d7d..ac89adbf 100644
--- a/rhodecode/templates/admin/users/user_edit_my_account.html
+++ b/rhodecode/templates/admin/users/user_edit_my_account.html
@@ -54,7 +54,7 @@
 	                    <label for="new_password">${_('New password')}:</label>
 	                </div>
 	                <div class="input">
-	                    ${h.password('new_password',class_="medium")}
+	                    ${h.password('new_password',class_="medium",autocomplete="off")}
 	                </div>
 	             </div>
 	            
diff --git a/rhodecode/tests/__init__.py b/rhodecode/tests/__init__.py
index c53fb41b..a82fbeec 100644
--- a/rhodecode/tests/__init__.py
+++ b/rhodecode/tests/__init__.py
@@ -7,6 +7,9 @@ command.
 This module initializes the application via ``websetup`` (`paster
 setup-app`) and provides the base testing objects.
 """
+import os
+from os.path import join as jn
+
 from unittest import TestCase
 
 from paste.deploy import loadapp
@@ -14,7 +17,7 @@ from paste.script.appinstall import SetupCommand
 from pylons import config, url
 from routes.util import URLGenerator
 from webtest import TestApp
-import os
+
 from rhodecode.model import meta
 import logging
 
@@ -35,7 +38,7 @@ __all__ = ['environ', 'url', 'TestController', 'TESTS_TMP_PATH', 'HG_REPO',
 environ = {}
 
 #SOME GLOBALS FOR TESTS
-TESTS_TMP_PATH = '/tmp'
+TESTS_TMP_PATH = jn('/', 'tmp')
 
 HG_REPO = 'vcs_test_hg'
 GIT_REPO = 'vcs_test_git'
@@ -64,8 +67,8 @@ class TestController(TestCase):
                                   'password':password})
 
         if 'invalid user name' in response.body:
-            assert False, 'could not login using %s %s' % (username, password)
+            self.fail('could not login using %s %s' % (username, password))
 
-        assert response.status == '302 Found', 'Wrong response code from login got %s' % response.status
-        assert response.session['rhodecode_user'].username == username, 'wrong logged in user got %s expected %s' % (response.session['rhodecode_user'].username, username)
+        self.assertEqual(response.status, '302 Found')
+        self.assertEqual(response.session['rhodecode_user'].username, username)
         return response.follow()
diff --git a/rhodecode/tests/functional/test_admin_repos.py b/rhodecode/tests/functional/test_admin_repos.py
index aa39d21e..abe691b9 100644
--- a/rhodecode/tests/functional/test_admin_repos.py
+++ b/rhodecode/tests/functional/test_admin_repos.py
@@ -6,6 +6,11 @@ from rhodecode.tests import *
 
 class TestAdminReposController(TestController):
 
+
+    def __make_repo(self):
+        pass
+
+
     def test_index(self):
         self.log_user()
         response = self.app.get(url('repos'))
@@ -21,32 +26,40 @@ class TestAdminReposController(TestController):
         private = False
         response = self.app.post(url('repos'), {'repo_name':repo_name,
                                                 'repo_type':'hg',
+                                                'clone_uri':'',
+                                                'repo_group':'',
                                                 'description':description,
                                                 'private':private})
-
+        self.assertTrue('flash' in response.session)
 
         #test if we have a message for that repository
-        assert '''created repository %s''' % (repo_name) in response.session['flash'][0], 'No flash message about new repo'
+        self.assertTrue('''created repository %s''' % (repo_name) in
+                        response.session['flash'][0])
 
-        #test if the fork was created in the database
-        new_repo = self.sa.query(Repository).filter(Repository.repo_name == repo_name).one()
+        #test if the repo was created in the database
+        new_repo = self.sa.query(Repository).filter(Repository.repo_name ==
+                                                    repo_name).one()
 
-        assert new_repo.repo_name == repo_name, 'wrong name of repo name in db'
-        assert new_repo.description == description, 'wrong description'
+        self.assertEqual(new_repo.repo_name, repo_name)
+        self.assertEqual(new_repo.description, description)
 
         #test if repository is visible in the list ?
         response = response.follow()
 
-        assert repo_name in response.body, 'missing new repo from the main repos list'
+        self.assertTrue(repo_name in response.body)
 
 
         #test if repository was created on filesystem
         try:
             vcs.get_repo(os.path.join(TESTS_TMP_PATH, repo_name))
         except:
-            assert False , 'no repo in filesystem'
+            self.fail('no repo in filesystem')
 
 
+    def test_create_hg_in_group(self):
+        #TODO: write test !
+        pass
+
     def test_create_git(self):
         return
         self.log_user()
@@ -55,6 +68,8 @@ class TestAdminReposController(TestController):
         private = False
         response = self.app.post(url('repos'), {'repo_name':repo_name,
                                                 'repo_type':'git',
+                                                'clone_uri':'',
+                                                'repo_group':'',
                                                 'description':description,
                                                 'private':private})
 
@@ -90,58 +105,74 @@ class TestAdminReposController(TestController):
         response = self.app.put(url('repo', repo_name=HG_REPO))
 
     def test_update_browser_fakeout(self):
-        response = self.app.post(url('repo', repo_name=HG_REPO), params=dict(_method='put'))
+        response = self.app.post(url('repo', repo_name=HG_REPO),
+                                 params=dict(_method='put'))
 
     def test_delete(self):
         self.log_user()
         repo_name = 'vcs_test_new_to_delete'
         description = 'description for newly created repo'
         private = False
+
         response = self.app.post(url('repos'), {'repo_name':repo_name,
                                                 'repo_type':'hg',
-                                               'description':description,
-                                               'private':private})
-
+                                                'clone_uri':'',
+                                                'repo_group':'',
+                                                'description':description,
+                                                'private':private})
+        self.assertTrue('flash' in response.session)
 
         #test if we have a message for that repository
-        assert '''created repository %s''' % (repo_name) in response.session['flash'][0], 'No flash message about new repo'
+        self.assertTrue('''created repository %s''' % (repo_name) in
+                        response.session['flash'][0])
 
         #test if the repo was created in the database
-        new_repo = self.sa.query(Repository).filter(Repository.repo_name == repo_name).one()
+        new_repo = self.sa.query(Repository).filter(Repository.repo_name ==
+                                                    repo_name).one()
 
-        assert new_repo.repo_name == repo_name, 'wrong name of repo name in db'
-        assert new_repo.description == description, 'wrong description'
+        self.assertEqual(new_repo.repo_name, repo_name)
+        self.assertEqual(new_repo.description, description)
 
         #test if repository is visible in the list ?
         response = response.follow()
 
-        assert repo_name in response.body, 'missing new repo from the main repos list'
+        self.assertTrue(repo_name in response.body)
 
 
         response = self.app.delete(url('repo', repo_name=repo_name))
 
-        assert '''deleted repository %s''' % (repo_name) in response.session['flash'][0], 'No flash message about delete repo'
+        self.assertTrue('''deleted repository %s''' % (repo_name) in
+                        response.session['flash'][0])
 
         response.follow()
 
         #check if repo was deleted from db
-        deleted_repo = self.sa.query(Repository).filter(Repository.repo_name == repo_name).scalar()
+        deleted_repo = self.sa.query(Repository).filter(Repository.repo_name
+                                                        == repo_name).scalar()
+
+        self.assertEqual(deleted_repo, None)
+
 
-        assert deleted_repo is None, 'Deleted repository was found in db'
+    def test_delete_repo_with_group(self):
+        #TODO:
+        pass
 
 
     def test_delete_browser_fakeout(self):
-        response = self.app.post(url('repo', repo_name=HG_REPO), params=dict(_method='delete'))
+        response = self.app.post(url('repo', repo_name=HG_REPO),
+                                 params=dict(_method='delete'))
 
     def test_show(self):
         self.log_user()
         response = self.app.get(url('repo', repo_name=HG_REPO))
 
     def test_show_as_xml(self):
-        response = self.app.get(url('formatted_repo', repo_name=HG_REPO, format='xml'))
+        response = self.app.get(url('formatted_repo', repo_name=HG_REPO,
+                                    format='xml'))
 
     def test_edit(self):
         response = self.app.get(url('edit_repo', repo_name=HG_REPO))
 
     def test_edit_as_xml(self):
-        response = self.app.get(url('formatted_edit_repo', repo_name=HG_REPO, format='xml'))
+        response = self.app.get(url('formatted_edit_repo', repo_name=HG_REPO,
+                                    format='xml'))
diff --git a/rhodecode/tests/functional/test_admin_settings.py b/rhodecode/tests/functional/test_admin_settings.py
index 4e8d9e42..2c2dc5dd 100644
--- a/rhodecode/tests/functional/test_admin_settings.py
+++ b/rhodecode/tests/functional/test_admin_settings.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 from rhodecode.lib.auth import get_crypt_password, check_password
 from rhodecode.model.db import User, RhodeCodeSettings
 from rhodecode.tests import *
@@ -42,7 +44,8 @@ class TestAdminSettingsController(TestController):
         response = self.app.get(url('admin_edit_setting', setting_id=1))
 
     def test_edit_as_xml(self):
-        response = self.app.get(url('formatted_admin_edit_setting', setting_id=1, format='xml'))
+        response = self.app.get(url('formatted_admin_edit_setting',
+                                    setting_id=1, format='xml'))
 
 
     def test_ga_code_active(self):
@@ -58,11 +61,14 @@ class TestAdminSettingsController(TestController):
                                                  rhodecode_ga_code=new_ga_code
                                                  ))
 
-        assert 'Updated application settings' in response.session['flash'][0][1], 'no flash message about success of change'
-        assert RhodeCodeSettings.get_app_settings()['rhodecode_ga_code'] == new_ga_code, 'change not in database'
+        self.assertTrue('Updated application settings' in
+                        response.session['flash'][0][1])
+        self.assertEqual(RhodeCodeSettings
+                         .get_app_settings()['rhodecode_ga_code'], new_ga_code)
 
         response = response.follow()
-        assert """_gaq.push(['_setAccount', '%s']);""" % new_ga_code in response.body
+        self.assertTrue("""_gaq.push(['_setAccount', '%s']);""" % new_ga_code
+                        in response.body)
 
     def test_ga_code_inactive(self):
         self.log_user()
@@ -77,11 +83,14 @@ class TestAdminSettingsController(TestController):
                                                  rhodecode_ga_code=new_ga_code
                                                  ))
 
-        assert 'Updated application settings' in response.session['flash'][0][1], 'no flash message about success of change'
-        assert RhodeCodeSettings.get_app_settings()['rhodecode_ga_code'] == new_ga_code, 'change not in database'
+        self.assertTrue('Updated application settings' in
+                        response.session['flash'][0][1])
+        self.assertEqual(RhodeCodeSettings
+                        .get_app_settings()['rhodecode_ga_code'], new_ga_code)
 
         response = response.follow()
-        assert """_gaq.push(['_setAccount', '%s']);""" % new_ga_code not in response.body
+        self.assertTrue("""_gaq.push(['_setAccount', '%s']);""" % new_ga_code
+                        not in response.body)
 
 
     def test_title_change(self):
@@ -89,27 +98,33 @@ class TestAdminSettingsController(TestController):
         old_title = 'RhodeCode'
         new_title = old_title + '_changed'
         old_realm = 'RhodeCode authentication'
-        response = self.app.post(url('admin_setting', setting_id='global'),
-                                     params=dict(
-                                                 _method='put',
-                                                 rhodecode_title=new_title,
-                                                 rhodecode_realm=old_realm,
-                                                 rhodecode_ga_code=''
-                                                 ))
 
+        for new_title in ['Changed', 'Żółwik', old_title]:
+            response = self.app.post(url('admin_setting', setting_id='global'),
+                                         params=dict(
+                                                     _method='put',
+                                                     rhodecode_title=new_title,
+                                                     rhodecode_realm=old_realm,
+                                                     rhodecode_ga_code=''
+                                                     ))
 
-        assert 'Updated application settings' in response.session['flash'][0][1], 'no flash message about success of change'
-        assert RhodeCodeSettings.get_app_settings()['rhodecode_title'] == new_title, 'change not in database'
 
-        response = response.follow()
-        assert """<h1><a href="/">%s</a></h1>""" % new_title in response.body
+            self.assertTrue('Updated application settings' in
+                            response.session['flash'][0][1])
+            self.assertEqual(RhodeCodeSettings
+                             .get_app_settings()['rhodecode_title'],
+                             new_title.decode('utf-8'))
+
+            response = response.follow()
+            self.assertTrue("""<h1><a href="/">%s</a></h1>""" % new_title
+                        in response.body)
 
 
     def test_my_account(self):
         self.log_user()
         response = self.app.get(url('admin_settings_my_account'))
-        print response
-        assert 'value="test_admin' in response.body
+
+        self.assertTrue('value="test_admin' in response.body)
 
     def test_my_account_update(self):
         self.log_user()
@@ -120,14 +135,14 @@ class TestAdminSettingsController(TestController):
         new_password = 'test123'
 
 
-        response = self.app.post(url('admin_settings_my_account_update'), params=dict(
-                                                            _method='put',
-                                                            username='test_admin',
-                                                            new_password=new_password,
-                                                            password='',
-                                                            name=new_name,
-                                                            lastname=new_lastname,
-                                                            email=new_email,))
+        response = self.app.post(url('admin_settings_my_account_update'),
+                                 params=dict(_method='put',
+                                             username='test_admin',
+                                             new_password=new_password,
+                                             password='',
+                                             name=new_name,
+                                             lastname=new_lastname,
+                                             email=new_email,))
         response.follow()
 
         assert 'Your account was updated successfully' in response.session['flash'][0][1], 'no flash message about success of change'
diff --git a/rhodecode/tests/functional/test_login.py b/rhodecode/tests/functional/test_login.py
index 6f9c47bd..12de4de3 100644
--- a/rhodecode/tests/functional/test_login.py
+++ b/rhodecode/tests/functional/test_login.py
@@ -45,11 +45,11 @@ class TestLoginController(TestController):
 
     def test_login_short_password(self):
         response = self.app.post(url(controller='login', action='index'),
-                                 {'username':'error',
-                                  'password':'test'})
-        assert response.status == '200 OK', 'Wrong response from login page'
+                                 {'username':'test_admin',
+                                  'password':'as'})
+        self.assertEqual(response.status, '200 OK')
         print response.body
-        assert 'Enter 6 characters or more' in response.body, 'No error password message in response'
+        self.assertTrue('Enter 3 characters or more' in response.body)
 
     def test_login_wrong_username_password(self):
         response = self.app.post(url(controller='login', action='index'),
diff --git a/rhodecode/tests/functional/test_summary.py b/rhodecode/tests/functional/test_summary.py
index cc43575a..d57d1028 100644
--- a/rhodecode/tests/functional/test_summary.py
+++ b/rhodecode/tests/functional/test_summary.py
@@ -6,22 +6,38 @@ class TestSummaryController(TestController):
 
     def test_index(self):
         self.log_user()
-        response = self.app.get(url(controller='summary', action='index', repo_name=HG_REPO))
+        response = self.app.get(url(controller='summary',
+                                    action='index', repo_name=HG_REPO))
 
         #repo type
-        assert """<img style="margin-bottom:2px" class="icon" title="Mercurial repository" alt="Mercurial repository" src="/images/icons/hgicon.png"/>""" in response.body
-        assert """<img style="margin-bottom:2px" class="icon" title="public repository" alt="public repository" src="/images/icons/lock_open.png"/>""" in response.body
+        self.assertTrue("""<img style="margin-bottom:2px" class="icon" """
+                        """title="Mercurial repository" alt="Mercurial """
+                        """repository" src="/images/icons/hgicon.png"/>"""
+                        in response.body)
+        self.assertTrue("""<img style="margin-bottom:2px" class="icon" """
+                        """title="public repository" alt="public """
+                        """repository" src="/images/icons/lock_open.png"/>"""
+                        in response.body)
 
         #codes stats
+        self._enable_stats()
 
 
-        self._enable_stats()
         invalidate_cache('get_repo_cached_%s' % HG_REPO)
-        response = self.app.get(url(controller='summary', action='index', repo_name=HG_REPO))
-        assert """var data = {"Python": 42, "Rst": 11, "Bash": 2, "Makefile": 1, "Batch": 1, "Ini": 1, "Css": 1};""" in response.body, 'wrong info about % of codes stats'
+        response = self.app.get(url(controller='summary', action='index',
+                                    repo_name=HG_REPO))
+
+        self.assertTrue("""var data = {"py": {"count": 42, "desc": """
+                        """["Python"]}, "rst": {"count": 11, "desc": """
+                        """["Rst"]}, "sh": {"count": 2, "desc": ["Bash"]}, """
+                        """"makefile": {"count": 1, "desc": ["Makefile", """
+                        """"Makefile"]}, "cfg": {"count": 1, "desc": ["Ini"]},"""
+                        """ "css": {"count": 1, "desc": ["Css"]}, "bat": """
+                        """{"count": 1, "desc": ["Batch"]}};"""
+                        in response.body)
 
         # clone url...
-        assert """<input type="text" id="clone_url" readonly="readonly" value="hg clone http://test_admin@localhost:80/%s" size="70"/>""" % HG_REPO in response.body
+        self.assertTrue("""<input type="text" id="clone_url" readonly="readonly" value="hg clone http://test_admin@localhost:80/%s" size="70"/>""" % HG_REPO in response.body)
 
 
     def _enable_stats(self):
diff --git a/test.ini b/test.ini
index 821f29df..5ffcaf5a 100644
--- a/test.ini
+++ b/test.ini
@@ -7,6 +7,7 @@
 
 [DEFAULT]
 debug = true
+pdebug = false
 ################################################################################
 ## Uncomment and replace with the address which should receive                ## 
 ## any error reports after application crash                                  ##