Changeset created on Mon May 30 16:31:56 CEST 2011 by Seek You Too Description: Upgrade to gcj 4.4 and libgcj10 Meresco components now also runs on gcj 4.4 and libgcj 10. Some dependencies are updated; Due to a new pyinotify which uses a logging modlue the logging module in meresco components is renamed to log Baseline version: https://meresco.svn.sourceforge.net/svnroot/meresco/meresco-components/tags/version_3.4.13 diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/build-deps.txt version_3.4.13.1/build-deps.txt --- version_3.4.13/build-deps.txt 2011-05-30 14:11:30.000000000 +0200 +++ version_3.4.13.1/build-deps.txt 2011-05-30 16:31:11.000000000 +0200 @@ -2,13 +2,23 @@ python-dev (<< 2.6) libglib2.0-dev (>= 2.16.6-2) libglib2.0-dev (<< 2.16.7) -g++-4.3 (>= 4.3.2-1.1) -g++-4.3 (<< 4.4) -gcj-4.3 (>= 4.3.2-2.CQ2) -gcj-4.3 (<< 4.3.3) -libgcj9-dev (>= 4.3.2-2.CQ2) -libgcj9-dev (<< 4.3.3) -liblucene-core2-gcj9 (>= 2.2.0) -liblucene-core2-gcj9 (<< 2.2.1) -liblucene-core2-gcj9-dev (>= 2.2.0) -liblucene-core2-gcj9-dev (<< 2.2.1) +#lenny:g++-4.3 (>= 4.3.2-1.1) +#lenny:g++-4.3 (<< 4.4) +#lenny:gcj-4.3 (>= 4.3.2-2.CQ2) +#lenny:gcj-4.3 (<< 4.3.3) +#lenny:libgcj9-dev (>= 4.3.2-2.CQ2) +#lenny:libgcj9-dev (<< 4.3.3) +#lenny:liblucene-core2-gcj9 (>= 2.2.0) +#lenny:liblucene-core2-gcj9 (<< 2.2.1) +#lenny:liblucene-core2-gcj9-dev (>= 2.2.0) +#lenny:liblucene-core2-gcj9-dev (<< 2.2.1) +#squeeze:g++-4.4 (>= 4.4.5-8) +#squeeze:g++-4.4 (<< 4.5) +#squeeze:gcj-4.4 (>= 4.4.5-2.CQ2) +#squeeze:gcj-4.4 (<< 4.4.6) +#squeeze:libgcj10-dev (>= 4.4.5-2.CQ2) +#squeeze:libgcj10-dev (<< 4.4.6) +#squeeze:liblucene-core2-gcj10 (>= 2.2.0) +#squeeze:liblucene-core2-gcj10 (<< 2.2.1) +#squeeze:liblucene-core2-gcj10-dev (>= 2.2.0) +#squeeze:liblucene-core2-gcj10-dev (<< 2.2.1) diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/deps.txt version_3.4.13.1/deps.txt --- version_3.4.13/deps.txt 2011-05-30 14:11:30.000000000 +0200 +++ version_3.4.13.1/deps.txt 2011-05-30 16:28:02.000000000 +0200 @@ -20,7 +20,10 @@ python-storage (<<5.2) python-weightless-core (>= 0.5) python-weightless-core (<< 0.6) -liblucene-core2-gcj9 (>= 2.2.0) -liblucene-core2-gcj9 (< 2.4.0) -libgcj9-0 +#lenny:liblucene-core2-gcj9 (>= 2.2.0) +#lenny:liblucene-core2-gcj9 (< 2.4.0) +#squeeze:liblucene-core2-gcj10 (>= 2.2.0) +#squeeze:liblucene-core2-gcj10 (< 2.4.0) +#lenny:libgcj9-0 +#squeeze:libgcj10 lsof diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/meresco/components/log/directorylog.py version_3.4.13.1/meresco/components/log/directorylog.py --- version_3.4.13/meresco/components/log/directorylog.py 1970-01-01 01:00:00.000000000 +0100 +++ version_3.4.13.1/meresco/components/log/directorylog.py 2011-05-30 16:21:41.000000000 +0200 @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +## begin license ## +# +# "Meresco Components" are components to build searchengines, repositories +# and archives, based on "Meresco Core". +# +# Copyright (C) 2006-2011 Seek You Too (CQ2) http://www.cq2.nl +# Copyright (C) 2006-2011 Stichting Kennisnet http://www.kennisnet.nl +# +# This file is part of "Meresco Components" +# +# "Meresco Components" is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# "Meresco Components" is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with "Meresco Components"; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +## end license ## + +from __future__ import with_statement +from time import strftime, gmtime +from os.path import join, isdir, isfile +from os import makedirs, listdir, remove + +NR_OF_FILES_KEPT = 14 + +# '2009-11-02T11:30:00Z 127.0.0.1 0.0K 1.000s /sru query=query&operation=searchRetrieve&version=1.1\n' +logtemplate = '%(strTimestamp)s %(ipAddress)s %(size).1fK %(duration).3fs %(path)s %(queryArguments)s\n' + +class DirectoryLog(object): + def __init__(self, logdir): + self._previousLog = None + self._logdir = logdir + if not isdir(self._logdir): + makedirs(self._logdir) + + def log(self, timestamp, path, ipAddress, size, duration, queryArguments): + strTimestamp=strftime('%Y-%m-%dT%H:%M:%SZ', gmtime(timestamp)) + logFilename = join(self._logdir, strftime('%Y-%m-%d-query.log', gmtime(timestamp))) + + if logFilename != self._previousLog: + logs = sorted(listdir(self._logdir)) + while len(logs) >= NR_OF_FILES_KEPT: + remove(join(self._logdir, logs[0])) + logs = sorted(listdir(self._logdir)) + + with open(logFilename, 'a') as f: + f.write(logtemplate % locals()) + + def logExists(self, logName): + return isfile(join(self._logdir, logName)) + + def listlogs(self): + return listdir(self._logdir) + + def getlog(self, logName): + logFilename = join(self._logdir, logName) + with open(logFilename) as f: + for line in f: + yield line + + diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/meresco/components/log/__init__.py version_3.4.13.1/meresco/components/log/__init__.py --- version_3.4.13/meresco/components/log/__init__.py 1970-01-01 01:00:00.000000000 +0100 +++ version_3.4.13.1/meresco/components/log/__init__.py 2011-05-30 16:24:24.000000000 +0200 @@ -0,0 +1,29 @@ +## begin license ## +# +# "Meresco Components" are components to build searchengines, repositories +# and archives, based on "Meresco Core". +# +# Copyright (C) 2011 Seek You Too (CQ2) http://www.cq2.nl +# Copyright (C) 2011 Stichting Kennisnet http://www.kennisnet.nl +# +# This file is part of "Meresco Components" +# +# "Meresco Components" is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# "Meresco Components" is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with "Meresco Components"; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +## end license ## + +from logfileserver import LogFileServer +from querylog import QueryLog, QueryLogHelperForSru, QueryLogHelper +from directorylog import DirectoryLog diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/meresco/components/log/logfileserver.py version_3.4.13.1/meresco/components/log/logfileserver.py --- version_3.4.13/meresco/components/log/logfileserver.py 1970-01-01 01:00:00.000000000 +0100 +++ version_3.4.13.1/meresco/components/log/logfileserver.py 2011-05-30 16:21:41.000000000 +0200 @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +## begin license ## +# +# "Meresco Components" are components to build searchengines, repositories +# and archives, based on "Meresco Core". +# +# Copyright (C) 2006-2011 Seek You Too (CQ2) http://www.cq2.nl +# Copyright (C) 2006-2011 Stichting Kennisnet http://www.kennisnet.nl +# +# This file is part of "Meresco Components" +# +# "Meresco Components" is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# "Meresco Components" is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with "Meresco Components"; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +## end license ## + +from meresco.components.http import utils as httputils +from meresco.components.http.utils import CRLF + +from cgi import escape as _escapeHtml + +def escapeHtml(aString): + return _escapeHtml(aString).replace('"','"') + +class LogFileServer(object): + """ + Serve the logs through http and show a filelisting when not + requesting a specific path. + """ + + def __init__(self, name, log, basepath): + self._indexPaths = ['', '/'] + self._basepath = basepath + self._log = log + self._name = name + + def handleRequest(self, path, **kwargs): + if path in [self._basepath + p for p in self._indexPaths]: + return self._generateIndex(path) + elif path.startswith(self._basepath) and self._log.logExists(self._logNameFromPath(path)): + return self._generateSingleLog(path) + else: + return (x for x in [httputils.notFoundHtml, "

FILE NOT FOUND

"]) + + def _generateIndex(self, path): + yield httputils.okHtml + yield HTML_PAGE_TOP % {'name': escapeHtml(self._name)} + for log in sorted(self._log.listlogs(), reverse=True): + yield '
  • %s
    • ' % (self._basepath, escapeHtml(log), escapeHtml(log)) + yield HTML_PAGE_BOTTOM + + def _generateSingleLog(self, path): + yield httputils.okPlainText + for line in self._log.getlog(self._logNameFromPath(path)): + yield line + + def _logNameFromPath(self, path): + return path.split('/')[-1] + + +HTML_PAGE_TOP = """ + + + "%(name)s" Logging + + +

    Logging - logfile listing

    + + +""" diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/meresco/components/log/querylog.py version_3.4.13.1/meresco/components/log/querylog.py --- version_3.4.13/meresco/components/log/querylog.py 1970-01-01 01:00:00.000000000 +0100 +++ version_3.4.13.1/meresco/components/log/querylog.py 2011-05-30 16:21:41.000000000 +0200 @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +## begin license ## +# +# "Meresco Components" are components to build searchengines, repositories +# and archives, based on "Meresco Core". +# +# Copyright (C) 2006-2011 Seek You Too (CQ2) http://www.cq2.nl +# Copyright (C) 2006-2011 Stichting Kennisnet http://www.kennisnet.nl +# +# This file is part of "Meresco Components" +# +# "Meresco Components" is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# "Meresco Components" is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with "Meresco Components"; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +## end license ## + +from meresco.core import Observable + +from time import time +from urllib import urlencode + +class QueryLog(Observable): + """ + Log incoming http queries with ip-address, path, size, timestamp, duration + """ + + def __init__(self, log, loggedPaths): + Observable.__init__(self) + self._log = log + self._loggedPaths = loggedPaths + + def handleRequest(self, Client, path, **kwargs): + if not any(path.startswith(p) for p in self._loggedPaths): + return self.all.handleRequest(Client=Client, path=path, **kwargs) + return self._handleRequest(Client=Client, path=path, **kwargs) + + def _handleRequest(self, Client, path, **kwargs): + __callstack_var_queryArguments__ = {} + + timestamp = self._time() + ipAddress = Client[0] + sizeInBytes = 0 + for response in self.all.handleRequest(Client=Client, path=path, **kwargs): + if hasattr(response, '__len__'): + sizeInBytes += len(response) + yield response + size = sizeInBytes / 1024.0 + duration = self._time() - timestamp + queryArguments = str(urlencode(sorted(__callstack_var_queryArguments__.items()), doseq=True)) + + self._log.log(timestamp, path, ipAddress, size, duration, queryArguments) + + def _time(self): + return time() + + def unknown(self, method, *args, **kwargs): + return self.all.unknown(method, *args, **kwargs) + + +SKIP_ARGS = ['sortBy', 'sortDescending'] + +def duplicatedInvalidArgPutInBySRUParse_GET_RID_OF_THAT_(key, kwargs): + return '_' in key and key.replace('_','-') in kwargs + +class QueryLogHelperForSru(Observable): + def searchRetrieve(self, **kwargs): + queryArguments = self.ctx.queryArguments + for key, value in kwargs.items(): + if duplicatedInvalidArgPutInBySRUParse_GET_RID_OF_THAT_(key, kwargs): + continue + if key in SKIP_ARGS: + continue + queryArguments[key] = value + return self.any.searchRetrieve(**kwargs) + +class QueryLogHelper(Observable): + def handleRequest(self, arguments, **kwargs): + queryArguments = self.ctx.queryArguments + queryArguments.update(arguments) + return self.all.handleRequest(arguments=arguments, **kwargs) Only in version_3.4.13/meresco/components: logging diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/test/_alltests.py version_3.4.13.1/test/_alltests.py --- version_3.4.13/test/_alltests.py 2011-05-30 14:11:29.000000000 +0200 +++ version_3.4.13.1/test/_alltests.py 2011-05-30 16:22:32.000000000 +0200 @@ -90,8 +90,8 @@ from http.timeddictionarytest import TimedDictionaryTest from http.utilstest import UtilsTest -from logging.logfileservertest import LogFileServerTest -from logging.querylogtest import QueryLogTest +from log.logfileservertest import LogFileServerTest +from log.querylogtest import QueryLogTest from numeric.converttest import ConvertTest from numeric.numbercomparitorfieldlettest import NumberComparitorFieldletTest diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/test/log/__init__.py version_3.4.13.1/test/log/__init__.py --- version_3.4.13/test/log/__init__.py 1970-01-01 01:00:00.000000000 +0100 +++ version_3.4.13.1/test/log/__init__.py 2011-05-30 16:21:23.000000000 +0200 @@ -0,0 +1,25 @@ +## begin license ## +# +# "Meresco Components" are components to build searchengines, repositories +# and archives, based on "Meresco Core". +# +# Copyright (C) 2011 Seek You Too (CQ2) http://www.cq2.nl +# Copyright (C) 2011 Stichting Kennisnet http://www.kennisnet.nl +# +# This file is part of "Meresco Components" +# +# "Meresco Components" is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# "Meresco Components" is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with "Meresco Components"; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +## end license ## diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/test/log/logfileservertest.py version_3.4.13.1/test/log/logfileservertest.py --- version_3.4.13/test/log/logfileservertest.py 1970-01-01 01:00:00.000000000 +0100 +++ version_3.4.13.1/test/log/logfileservertest.py 2011-05-30 16:22:58.000000000 +0200 @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +## begin license ## +# +# "Meresco Components" are components to build searchengines, repositories +# and archives, based on "Meresco Core". +# +# Copyright (C) 2006-2011 Seek You Too (CQ2) http://www.cq2.nl +# Copyright (C) 2006-2011 Stichting Kennisnet http://www.kennisnet.nl +# +# This file is part of "Meresco Components" +# +# "Meresco Components" is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# "Meresco Components" is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with "Meresco Components"; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +## end license ## + +from cq2utils import CQ2TestCase, CallTrace + +from os.path import join +from os import mkdir, listdir + +from meresco.components.http import utils as httputils +from meresco.components.http.utils import CRLF, notFoundHtml + +from meresco.components.log import LogFileServer, DirectoryLog + +class LogFileServerTest(CQ2TestCase): + def setUp(self): + CQ2TestCase.setUp(self) + + self.logDir = join(self.tempdir, 'log') + directoryLog = DirectoryLog(self.logDir) + self.qlfs = LogFileServer("Fancy ", directoryLog, basepath='/log') + + def testGenerateEmptyHtmlFileLinkListing(self): + headers, body = "".join(self.qlfs.handleRequest(path="/log")).split(CRLF+CRLF) + + self.assertEquals('HTTP/1.0 200 OK\r\nContent-Type: text/html; charset=utf-8', headers) + self.assertTrue(body.startswith('\n'), body) + self.assertTrue(body.rfind('\n') != -1, body) + + self.assertTrue('"Fancy <name>" Logging' in body, body) + self.assertTrue('Logging - logfile listing' in body, body) + + def testEmptyDirectoryEmptyHtmlResult(self): + headers, body = "".join(self.qlfs.handleRequest(path="/")).split(CRLF+CRLF) + self.assertFalse('
  • ' in body) + + def testDirectoryHtmlResult(self): + filename = '2009-11-10-afile.1' + open(join(self.logDir, filename), 'w').close() + + headers, body = "".join(self.qlfs.handleRequest(path="/log")).split(CRLF+CRLF) + self.assertTrue('
  • ' in body) + self.assertTrue(' body.index(filename2), 'The files should be sorted.') + + def testPathNotSpecifiedAsIndexEffectivelyUsesMerescoFileServer(self): + headers, body = "".join(self.qlfs.handleRequest(path="/thisIsWrongMister")).split(CRLF+CRLF) + self.assertTrue("HTTP/1.0 404 Not Found" in headers, headers) + + def testPathIsASubDir(self): + aSubDirCalled = "subdir" + mkdir(join(self.logDir, aSubDirCalled)) + + headers, body = "".join(self.qlfs.handleRequest(path="/%s" % aSubDirCalled)).split(CRLF+CRLF) + + self.assertTrue("HTTP/1.0 404 Not Found" in headers, headers) + + def testGetNonExistingLogFile(self): + headers, body = "".join(self.qlfs.handleRequest(path="/log/thisIsWrongMister")).split(CRLF+CRLF) + self.assertTrue("HTTP/1.0 404 Not Found" in headers, headers) diff --unidirectional-new-file --exclude='*.so' --exclude='*.o' --exclude=.svn --exclude='*.pyc' --exclude=deps.d --exclude=applied --recursive --unified version_3.4.13/test/log/querylogtest.py version_3.4.13.1/test/log/querylogtest.py --- version_3.4.13/test/log/querylogtest.py 1970-01-01 01:00:00.000000000 +0100 +++ version_3.4.13.1/test/log/querylogtest.py 2011-05-30 16:24:59.000000000 +0200 @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +## begin license ## +# +# "Meresco Components" are components to build searchengines, repositories +# and archives, based on "Meresco Core". +# +# Copyright (C) 2006-2011 Seek You Too (CQ2) http://www.cq2.nl +# Copyright (C) 2006-2011 Stichting Kennisnet http://www.kennisnet.nl +# +# This file is part of "Meresco Components" +# +# "Meresco Components" is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# "Meresco Components" is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with "Meresco Components"; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +## end license ## + +from cq2utils import CQ2TestCase, CallTrace + +from os.path import isfile, isdir, join +from os import listdir + +from meresco.components.log import QueryLog, QueryLogHelper, QueryLogHelperForSru, DirectoryLog +from meresco.components.log.directorylog import NR_OF_FILES_KEPT + +from meresco.core import Observable + +class QueryLogTest(CQ2TestCase): + def setUp(self): + CQ2TestCase.setUp(self) + self._timeNow = 1257161136.0 # 2009-11-02 11:30:00 + def time(): + self._timeNow += 1.0 + return self._timeNow + directoryLog = DirectoryLog(self.tempdir) + self.queryLog = QueryLog(log=directoryLog, loggedPaths=['/edurep/sru', '/edurep/srw']) + self.queryLog._time = time + + def testLogging(self): + observer = CallTrace('observer') + observer.returnValues['handleRequest'] = (line for line in ['1','2','3']) + self.queryLog.addObserver(observer) + result = ''.join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/edurep/sru', otherArg='value')) + + self.assertEquals('123', result) + self.assertEquals(['handleRequest'], [m.name for m in observer.calledMethods]) + self.assertEquals([dict(Client=('127.0.0.1', 47785), path='/edurep/sru', otherArg='value')], [m.kwargs for m in observer.calledMethods]) + + self.assertTrue(isfile(join(self.tempdir, '2009-11-02-query.log'))) + self.assertEquals('2009-11-02T11:25:37Z 127.0.0.1 0.0K 1.000s /edurep/sru \n', open(join(self.tempdir, '2009-11-02-query.log')).read()) + + def testAddToLogfile(self): + f = open(join(self.tempdir, '2009-11-02-query.log'), 'w') + f.write('line0\n') + f.close() + observer = CallTrace('observer') + observer.returnValues['handleRequest'] = (line for line in ['1','2','3']) + self.queryLog.addObserver(observer) + ''.join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/edurep/sru', otherArg='value')) + + self.assertEquals(2, len(open(join(self.tempdir, '2009-11-02-query.log')).readlines())) + + def testLogCanReturnCallables(self): + observer= CallTrace('observer') + observer.returnValues['handleRequest'] = (f for f in ['1', lambda: None,'3']) + self.queryLog.addObserver(observer) + list(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/edurep/sru', otherArg='value')) + + self.assertEquals(1, len(open(join(self.tempdir, '2009-11-02-query.log')).readlines())) + + def testNewDayNewLogfile(self): + observer = CallTrace('observer') + observer.returnValues['handleRequest'] = (line for line in ['1','2','3']) + self.queryLog.addObserver(observer) + ''.join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/edurep/sru', otherArg='value')) + self._timeNow += 24 * 60 *60 + observer.returnValues['handleRequest'] = (line for line in ['1','2','3']) + ''.join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/edurep/sru', otherArg='value')) + + self.assertEquals(['2009-11-02-query.log', '2009-11-03-query.log'], sorted(listdir(self.tempdir))) + + def testIncludedPathsOnly(self): + observer = CallTrace('observer') + observer.returnValues['handleRequest'] = (line for line in ['1','2','3']) + self.queryLog.addObserver(observer) + result = ''.join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/not/included', otherArg='value')) + + self.assertEquals('123', result) + self.assertEquals(0, len(listdir(self.tempdir))) + + + def testLoggedPathsIsStartOfAcceptedPath(self): + observer = CallTrace('observer') + observer.returnValues['handleRequest'] = (line for line in ['1','2','3']) + self.queryLog.addObserver(observer) + ''.join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/edurep/sru/extended/path', otherArg='value')) + self.assertEquals(1, len(listdir(self.tempdir))) + + def testLogDirCreated(self): + logDir = join(self.tempdir, 'amihere') + self.assertFalse(isdir(logDir)) + + queryLog = QueryLog(log=DirectoryLog(logDir), loggedPaths=None) + self.assertTrue(isdir(logDir)) + + def testLogQueryParameters(self): + class HandleRequestObserver(Observable): + def handleRequest(self, **kwargs): + self.ctx.queryArguments.update({'a':'A', 'b':'B', 'c':'C', 'd':['D','DD']}) + yield 'result' + self.queryLog.addObserver(HandleRequestObserver()) + result = ''.join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/edurep/sru', otherArg='value')) + self.assertEquals('result', result) + self.assertEquals('2009-11-02T11:25:37Z 127.0.0.1 0.0K 1.000s /edurep/sru a=A&b=B&c=C&d=D&d=DD\n', open(join(self.tempdir, '2009-11-02-query.log')).read()) + + def testQueryLogHelperForSru(self): + __callstack_var_queryArguments__ = {} + helper = QueryLogHelperForSru() + observer = CallTrace('observer') + helper.addObserver(observer) + observer.returnValues['searchRetrieve'] = 'result' + helper.searchRetrieve(query=['query'], x_term_drilldown='drilldown', sortBy='field', sortDescending=False, **{'x-term-drilldown':'drilldown', 'under_score':'value', 'sortKeys':'field,,0'}) + self.assertEquals({'query': ['query'], 'x-term-drilldown': 'drilldown', 'under_score': 'value', 'sortKeys':'field,,0'}, __callstack_var_queryArguments__) + + def testQueryLogHelper(self): + __callstack_var_queryArguments__ = {} + helper = QueryLogHelper() + observer = CallTrace('observer') + helper.addObserver(observer) + observer.returnValues['handleRequest'] = 'result' + result = list(helper.handleRequest(arguments={'key':['value'], 'key2':['value1', 'value2']}, path='path')) + self.assertEquals(['result'], result) + self.assertEquals({'key':['value'], 'key2':['value1', 'value2']}, __callstack_var_queryArguments__) + self.assertEquals([{'arguments': {'key':['value'], 'key2':['value1', 'value2']}, 'path':'path'}], [m.kwargs for m in observer.calledMethods]) + + + def testRemoveOldLogs(self): + for filename in ("%03d" % r for r in range(NR_OF_FILES_KEPT)): + open(join(self.tempdir, filename), 'w').close() + + filesBefore = listdir(self.tempdir) + "".join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path="/edurep/sru")) + filesAfter = listdir(self.tempdir) + self.assertFalse('000' in filesAfter) + self.assertEquals(len(filesAfter), len(filesBefore)) + + filesBefore = listdir(self.tempdir) + self._timeNow += 3600*24 + "".join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path="/edurep/sru")) + filesAfter = listdir(self.tempdir) + self.assertFalse('001' in filesAfter) + self.assertEquals(len(filesAfter), len(filesBefore)) + + open(join(self.tempdir, '015'), 'w').close() + open(join(self.tempdir, '016'), 'w').close() + self._timeNow += 3600*24 + "".join(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path="/edurep/sru")) + self.assertEquals(NR_OF_FILES_KEPT, len(listdir(self.tempdir))) + + + Only in version_3.4.13/test: logging