diff options
| author | Qt by Nokia <qt-info@nokia.com> | 2011-04-27 12:05:43 +0200 |
|---|---|---|
| committer | axis <qt-info@nokia.com> | 2011-04-27 12:05:43 +0200 |
| commit | 50123887ba0f33cf47520bee7c419d68742af2d1 (patch) | |
| tree | 0eb8679b9e4e4370e59b44bfdcae616816e39aca /3rdparty/clucene/src/CLucene/store | |
Initial import from the monolithic Qt.
This is the beginning of revision history for this module. If you
want to look at revision history older than this, please refer to the
Qt Git wiki for how to use Git history grafting. At the time of
writing, this wiki is located here:
http://qt.gitorious.org/qt/pages/GitIntroductionWithQt
If you have already performed the grafting and you don't see any
history beyond this commit, try running "git log" with the "--follow"
argument.
Branched from the monolithic repo, Qt master branch, at commit
896db169ea224deb96c59ce8af800d019de63f12
Diffstat (limited to '3rdparty/clucene/src/CLucene/store')
16 files changed, 3033 insertions, 0 deletions
diff --git a/3rdparty/clucene/src/CLucene/store/Directory.h b/3rdparty/clucene/src/CLucene/store/Directory.h new file mode 100644 index 000000000..818bc7af9 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/Directory.h @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_Directory +#define _lucene_store_Directory + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QStringList> + +#include "CLucene/store/Lock.h" +#include "CLucene/util/VoidList.h" +#include "CLucene/util/Misc.h" + +#include "IndexInput.h" +#include "IndexOutput.h" + +CL_NS_DEF(store) + +/** A Directory is a flat list of files. Files may be written once, when they +* are created. Once a file is created it may only be opened for read, or +* deleted. Random access is permitted both when reading and writing. +* +* <p> Direct i/o is not used directly, but rather all i/o is +* through this API. This permits things such as: <ul> +* <li> implementation of RAM-based indices; +* <li> implementation indices stored in a database, via a database; +* <li> implementation of an index as a single file; +* </ul> +* +*/ +class Directory : LUCENE_REFBASE +{ +protected: + Directory() {} + // Removes an existing file in the directory. + virtual bool doDeleteFile(const QString& name) = 0; + +public: + DEFINE_MUTEX(THIS_LOCK) + + virtual ~Directory() {}; + + // Returns an array of strings, one for each file in the directory. + virtual QStringList list() const = 0; + + // Returns true iff a file with the given name exists. + virtual bool fileExists(const QString& name) const = 0; + + // Returns the time the named file was last modified. + virtual int64_t fileModified(const QString& name) const = 0; + + // Returns the length of a file in the directory. + virtual int64_t fileLength(const QString& name) const = 0; + + // Returns a stream reading an existing file. + virtual IndexInput* openInput(const QString& name) = 0; + virtual IndexInput* openInput(const QString& name, int32_t bufferSize) + { + // didnt overload bufferSize + return openInput(name); + } + + // Set the modified time of an existing file to now. + virtual void touchFile(const QString& name) = 0; + + // Removes an existing file in the directory. + virtual bool deleteFile(const QString& name, const bool throwError = true) { + bool ret = doDeleteFile(name); + if (!ret && throwError) { + char buffer[200]; + _snprintf(buffer, 200, "couldn't delete file %s", + name.toLocal8Bit().constData()); + _CLTHROWA(CL_ERR_IO, buffer); + } + return ret; + } + + // Renames an existing file in the directory. + // If a file already exists with the new name, then it is replaced. + virtual void renameFile(const QString& from, const QString& to) = 0; + + // Creates a new, empty file in the directory with the given name. + // Returns a stream writing this file. + virtual IndexOutput* createOutput(const QString& name) = 0; + + // Construct a {@link Lock}. + // @param name the name of the lock file + virtual LuceneLock* makeLock(const QString& name) = 0; + + // Closes the store. + virtual void close() = 0; + + virtual QString toString() const = 0; + + virtual QString getDirectoryType() const = 0; +}; + +CL_NS_END + +#endif diff --git a/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp b/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp new file mode 100644 index 000000000..5f96e91cd --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp @@ -0,0 +1,662 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#include <QtCore/QDir> +#include <QtCore/QDateTime> +#include <QtCore/QFileInfo> +#include <QtCore/QByteArray> +#include <QtCore/QCryptographicHash> + +#include "CLucene/StdHeader.h" +#include "FSDirectory.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +CL_NS_DEF(store) +CL_NS_USE(util) + +bool FSDirectory::disableLocks = false; + +// This cache of directories ensures that there is a unique Directory instance +// per path, so that synchronization on the Directory can be used to synchronize +// access between readers and writers. +static CL_NS(util)::CLHashMap<QString, FSDirectory*, + CL_NS(util)::Compare::Qstring, CL_NS(util)::Equals::Qstring, + CL_NS(util)::Deletor::DummyQString> DIRECTORIES(false, false); + +// # pragma mark -- FSDirectory::FSLock + +FSDirectory::FSLock::FSLock(const QString& _lockDir, const QString& name) + : lockDir(_lockDir) + , lockFile(_lockDir + QDir::separator() + name) +{ +} + +FSDirectory::FSLock::~FSLock() +{ +} + +bool FSDirectory::FSLock::obtain() +{ + if (disableLocks) + return true; + + if (QFile::exists(lockFile)) + return false; + + QDir dir(lockDir); + if (!dir.exists()) { + if (!dir.mkpath(lockDir)) { + // 34: len of "Couldn't create lock directory: " + char* err = _CL_NEWARRAY( + char, 34 + strlen(lockDir.toLocal8Bit().constData()) + 1); + strcpy(err, "Couldn't create lock directory: "); + strcat(err, lockDir.toLocal8Bit().constData()); + _CLTHROWA_DEL(CL_ERR_IO, err); + } + } + + QFile file(lockFile); + return file.open(QIODevice::ReadWrite); +} + +void FSDirectory::FSLock::release() +{ + if (disableLocks) + return; + + QFile file(lockFile); + file.remove(); +} + +bool FSDirectory::FSLock::isLocked() +{ + if (disableLocks) + return false; + return QFile::exists(lockFile); +} + +QString FSDirectory::FSLock::toString() const +{ + QString ret(QLatin1String("Lock@")); + return ret.append(lockFile); +} + +// # pragma mark -- FSDirectory::FSIndexInput + +FSDirectory::FSIndexInput::FSIndexInput(const QString& path, int32_t bufferSize) + : BufferedIndexInput(bufferSize) +{ + CND_PRECONDITION(!path.isEmpty(), "path is NULL"); + + handle = _CLNEW SharedHandle(); + handle->fhandle.setFileName(path); + handle->fhandle.open(QIODevice::ReadOnly); + + if (handle->fhandle.error() != QFile::NoError) { + switch(handle->fhandle.error()) { + case 1: + _CLTHROWA(CL_ERR_IO, "An error occurred when reading from the file"); + break; + case 2: + _CLTHROWA(CL_ERR_IO, "An error occurred when writing to the file."); + break; + case 5: + _CLTHROWA(CL_ERR_IO, "The file could not be opened."); + break; + case 6: + _CLTHROWA(CL_ERR_IO, "The operation was aborted."); + break; + case 7: + _CLTHROWA(CL_ERR_IO, "A timeout occurred."); + break; + case 8: + _CLTHROWA(CL_ERR_IO, "An unspecified error occurred."); + break; + case 9: + _CLTHROWA(CL_ERR_IO, "The file could not be removed."); + break; + case 10: + _CLTHROWA(CL_ERR_IO, "The file could not be renamed."); + break; + case 11: + _CLTHROWA(CL_ERR_IO, "The position in the file could not be changed."); + break; + case 12: + _CLTHROWA(CL_ERR_IO, "The file could not be resized.e"); + break; + case 13: + _CLTHROWA(CL_ERR_IO, "The file could not be accessed."); + break; + case 14: + _CLTHROWA(CL_ERR_IO, "The file could not be copied."); + break; + case 4: + default: + _CLTHROWA(CL_ERR_IO, "A fatal error occurred."); + } + } + + //Store the file length + handle->_length = handle->fhandle.size(); + handle->_fpos = 0; + this->_pos = 0; +} + +FSDirectory::FSIndexInput::FSIndexInput(const FSIndexInput& other) + : BufferedIndexInput(other) +{ + if (other.handle == NULL) + _CLTHROWA(CL_ERR_NullPointer, "other handle is null"); + + SCOPED_LOCK_MUTEX(*other.handle->THIS_LOCK) + + _pos = other.handle->_fpos; + handle = _CL_POINTER(other.handle); +} + +FSDirectory::FSIndexInput::~FSIndexInput() +{ + FSIndexInput::close(); +} + +void FSDirectory::FSIndexInput::close() +{ + BufferedIndexInput::close(); +#ifdef _LUCENE_THREADMUTEX + if (handle != NULL) { + // Here we have a bit of a problem... We need to lock the handle to + // ensure that we can safely delete the handle... But if we delete the + // handle, then the scoped unlock, won't be able to unlock the mutex... + + // take a reference of the lock object... + _LUCENE_THREADMUTEX* mutex = handle->THIS_LOCK; + //lock the mutex + mutex->lock(); + + // determine if we are about to delete the handle... + bool doUnlock = (handle->__cl_refcount > 1); + // decdelete (deletes if refcount is down to 0) + _CLDECDELETE(handle); + + if (doUnlock) + mutex->unlock(); + else + delete mutex; + } +#else + _CLDECDELETE(handle); +#endif +} + +IndexInput* FSDirectory::FSIndexInput::clone() const +{ + return _CLNEW FSDirectory::FSIndexInput(*this); +} + +void FSDirectory::FSIndexInput::seekInternal(const int64_t position) +{ + CND_PRECONDITION(position >= 0 && position < handle->_length, + "Seeking out of range") + _pos = position; +} + +void FSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) +{ + SCOPED_LOCK_MUTEX(*handle->THIS_LOCK) + + CND_PRECONDITION(handle != NULL, "shared file handle has closed"); + CND_PRECONDITION(handle->fhandle.isOpen(), "file is not open"); + + if (handle->_fpos != _pos) { + handle->fhandle.seek(_pos); + if (handle->fhandle.pos() != _pos) + _CLTHROWA( CL_ERR_IO, "File IO Seek error"); + handle->_fpos = _pos; + } + + bufferLength = (int32_t)handle->fhandle.read((char*)b, len); + if (bufferLength == 0) + _CLTHROWA(CL_ERR_IO, "read past EOF"); + + if (bufferLength == -1) + _CLTHROWA(CL_ERR_IO, "read error"); + + _pos += bufferLength; + handle->_fpos =_pos; +} + +// # pragma mark -- FSDirectory::FSIndexInput::SharedHandle + +FSDirectory::FSIndexInput::SharedHandle::SharedHandle() + : _fpos(0) + , _length(0) +{ +#ifdef _LUCENE_THREADMUTEX + THIS_LOCK = new _LUCENE_THREADMUTEX; +#endif +} + +FSDirectory::FSIndexInput::SharedHandle::~SharedHandle() +{ + if (fhandle.isOpen()) + fhandle.close(); +} + +// # pragma mark -- FSDirectory::FSIndexOutput + +FSDirectory::FSIndexOutput::FSIndexOutput(const QString& path) +{ + //O_BINARY - Opens file in binary (untranslated) mode + //O_CREAT - Creates and opens new file for writing. Has no effect if file specified by filename exists + //O_RANDOM - Specifies that caching is optimized for, but not restricted to, random access from disk. + //O_WRONLY - Opens file for writing only; + fhandle.setFileName(path); + fhandle.open(QIODevice::ReadWrite | QIODevice::Truncate); + + if (fhandle.error() != QFile::NoError) { + switch(fhandle.error()) { + case 1: + _CLTHROWA(CL_ERR_IO, "An error occurred when reading from the file"); + break; + case 2: + _CLTHROWA(CL_ERR_IO, "An error occurred when writing to the file."); + break; + case 5: + _CLTHROWA(CL_ERR_IO, "The file could not be opened."); + break; + case 6: + _CLTHROWA(CL_ERR_IO, "The operation was aborted."); + break; + case 7: + _CLTHROWA(CL_ERR_IO, "A timeout occurred."); + break; + case 8: + _CLTHROWA(CL_ERR_IO, "An unspecified error occurred."); + break; + case 9: + _CLTHROWA(CL_ERR_IO, "The file could not be removed."); + break; + case 10: + _CLTHROWA(CL_ERR_IO, "The file could not be renamed."); + break; + case 11: + _CLTHROWA(CL_ERR_IO, "The position in the file could not be changed."); + break; + case 12: + _CLTHROWA(CL_ERR_IO, "The file could not be resized.e"); + break; + case 13: + _CLTHROWA(CL_ERR_IO, "The file could not be accessed."); + break; + case 14: + _CLTHROWA(CL_ERR_IO, "The file could not be copied."); + break; + case 4: + default: + _CLTHROWA(CL_ERR_IO, "A fatal error occurred."); + } + } +} + +FSDirectory::FSIndexOutput::~FSIndexOutput() +{ + if (fhandle.isOpen()) { + try { + FSIndexOutput::close(); + } catch (CLuceneError& err) { + //ignore IO errors... + if (err.number() != CL_ERR_IO) + throw; + } + } +} + +void FSDirectory::FSIndexOutput::close() +{ + try { + BufferedIndexOutput::close(); + } catch (CLuceneError& err) { + //ignore IO errors... + if (err.number() != CL_ERR_IO) + throw; + } + fhandle.close(); +} + +int64_t FSDirectory::FSIndexOutput::length() +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + return fhandle.size(); +} + +void FSDirectory::FSIndexOutput::seek(const int64_t pos) +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + + BufferedIndexOutput::seek(pos); + fhandle.seek(pos); + if (fhandle.pos() != pos) + _CLTHROWA(CL_ERR_IO, "File IO Seek error"); +} + +void FSDirectory::FSIndexOutput::flushBuffer(const uint8_t* b, const int32_t size) +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + + if (size > 0 && fhandle.write((const char*)b, size) != size) + _CLTHROWA(CL_ERR_IO, "File IO Write error"); +} + +// # pragma mark -- FSDirectory + +FSDirectory::FSDirectory(const QString& path, const bool createDir) + : Directory() + , refCount(0) + , useMMap(false) +{ + //set a realpath so that if we change directory, we can still function + directory = QFileInfo(path).absoluteFilePath(); + lockDir = directory; + + QDir dir(lockDir); + if (!dir.exists()) { + if (!dir.mkpath(lockDir)) + _CLTHROWA_DEL(CL_ERR_IO, "Cannot create temp directory"); + } + + QFileInfo info(lockDir); + if (info.isFile() || info.isSymLink()) + _CLTHROWA(CL_ERR_IO, "Found regular file where directory expected"); + + if (createDir) + create(); + + dir.setPath(directory); + if (!dir.exists()) { + //19: len of " is not a directory" + char* err = + _CL_NEWARRAY(char, 19 + strlen(path.toLocal8Bit().constData()) + 1); + strcpy(err, path.toLocal8Bit().constData()); + strcat(err, " is not a directory"); + _CLTHROWA_DEL(CL_ERR_IO, err); + } +} + +void FSDirectory::create() +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + + bool clear = false; + QDir dir(directory); + if (!dir.exists()) { + if (!dir.mkpath(directory)) { + char* err = _CL_NEWARRAY( // 27 len of "Couldn't create directory:" + char, 27 + strlen(directory.toLocal8Bit().constData()) + 1); + strcpy(err, "Couldn't create directory: "); + strcat(err, directory.toLocal8Bit().constData()); + _CLTHROWA_DEL(CL_ERR_IO, err); + } + } else { + clear = true; + } + + QFileInfo info(directory); + if (info.isFile() || info.isSymLink()) { + char tmp[1024]; + _snprintf(tmp, 1024, "%s not a directory", + directory.toLocal8Bit().constData()); + _CLTHROWA(CL_ERR_IO, tmp); + } + + if (clear) { + dir.setPath(directory); + // clear probably existing lucene index files + QStringList fileList = dir.entryList(QDir::Files | QDir::Hidden + | QDir::NoSymLinks); + foreach(const QString file, fileList) { + if (CL_NS(index)::IndexReader::isLuceneFile(file)) { + if (!dir.remove(file)) + _CLTHROWA(CL_ERR_IO, "Couldn't delete file "); + } + } + + // clear probably existing file locks + QFileInfo dirInfo(lockDir); + if (dirInfo.exists() && dirInfo.isReadable() && dirInfo.isWritable() + && !dirInfo.isFile() && !dirInfo.isSymLink()) { + QDir lockDirectory(lockDir); + fileList = dir.entryList(QStringList() << getLockPrefix() + + QLatin1Char('*'), QDir::Files | QDir::Hidden | QDir::NoSymLinks); + + foreach(const QString file, fileList) { + if (!lockDirectory.remove(file)) + _CLTHROWA(CL_ERR_IO, "Couldn't delete file "); + } + } + else { + //todo: richer error: + lockDir.getAbsolutePath()); + _CLTHROWA(CL_ERR_IO, "Cannot read lock directory"); + } + } +} + +void FSDirectory::priv_getFN(QString& buffer, const QString& name) const +{ + buffer.clear(); + buffer.append(directory); + buffer.append(QDir::separator()); + buffer.append(name); +} + +FSDirectory::~FSDirectory() +{ +} + +QStringList FSDirectory::list() const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.entryList(QDir::Files | QDir::Hidden); +} + +bool FSDirectory::fileExists(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.entryList().contains(name); +} + +QString FSDirectory::getDirName() const +{ + return directory; +} + +//static +FSDirectory* FSDirectory::getDirectory(const QString& file, const bool _create) +{ + FSDirectory* dir = NULL; + { + if (file.isEmpty()) + _CLTHROWA(CL_ERR_IO, "Invalid directory"); + + SCOPED_LOCK_MUTEX(DIRECTORIES.THIS_LOCK) + dir = DIRECTORIES.get(file); + if ( dir == NULL ){ + dir = _CLNEW FSDirectory(file, _create); + DIRECTORIES.put(dir->directory, dir); + } else if (_create) { + dir->create(); + } + + { + SCOPED_LOCK_MUTEX(dir->THIS_LOCK) + dir->refCount++; + } + } + + return _CL_POINTER(dir); +} + +int64_t FSDirectory::fileModified(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFileInfo fInfo(directory + QDir::separator() + name); + return fInfo.lastModified().toTime_t(); +} + +//static +int64_t FSDirectory::fileModified(const QString& dir, const QString& name) +{ + QFileInfo fInfo(dir + QDir::separator() + name); + return fInfo.lastModified().toTime_t(); +} + +void FSDirectory::touchFile(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFile file(directory + QDir::separator() + name); + if (!file.open(QIODevice::ReadWrite)) + _CLTHROWA(CL_ERR_IO, "IO Error while touching file"); +} + +int64_t FSDirectory::fileLength(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFileInfo fInfo(directory + QDir::separator() + name); + return fInfo.size(); +} + +IndexInput* FSDirectory::openInput(const QString& name) +{ + return openInput(name, CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); +} + +IndexInput* FSDirectory::openInput(const QString& name, int32_t bufferSize ) +{ + CND_PRECONDITION(directory[0]!=0,"directory is not open") + + return _CLNEW FSIndexInput(directory + QDir::separator() + name, bufferSize); +} + +void FSDirectory::close() +{ + SCOPED_LOCK_MUTEX(DIRECTORIES.THIS_LOCK) + { + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + //refcount starts at 1 + if (--refCount <= 0) { + Directory* dir = DIRECTORIES.get(getDirName()); + if (dir) { + //this will be removed in ~FSDirectory + DIRECTORIES.remove(getDirName()); + _CLDECDELETE(dir); + } + } + } +} + +QString FSDirectory::getLockPrefix() const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QString dirName(QFileInfo(directory).absoluteFilePath()); + if (dirName.isEmpty()) + _CLTHROWA(CL_ERR_Runtime, "Invalid directory path"); + + // to be compatible with jlucene, + // we need to make some changes ... + if (dirName.at(1) == QLatin1Char(':')) + dirName[0] = dirName.at(0).toUpper(); + + TCHAR tBuffer[2048] = { 0 }; + dirName.toWCharArray(tBuffer); + + char aBuffer[4096] = { 0 }; + STRCPY_TtoA(aBuffer, tBuffer, 4096); + + QString string(QLatin1String("lucene-")); + QByteArray hash(QCryptographicHash::hash(aBuffer, QCryptographicHash::Md5)); + + // TODO: verify this !!! + return string.append(QLatin1String(hash.toHex().constData())); +} + +bool FSDirectory::doDeleteFile(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.remove(name); +} + +void FSDirectory::renameFile(const QString& from, const QString& to) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + SCOPED_LOCK_MUTEX(THIS_LOCK) + + if (fileExists(to)) + deleteFile(to, false); + + QFile file(directory + QDir::separator() + from); + QString newFile(directory + QDir::separator() + to); + if (!file.rename(newFile)) { + // try a second time if we fail + if (fileExists(to)) + deleteFile(to, false); + + if (!file.rename(newFile)) { + QString error(QLatin1String("Could not rename: %1 to %2!!!!")); + error.arg(from).arg(newFile); + QByteArray bArray(error.toLocal8Bit()); + _CLTHROWA(CL_ERR_IO, bArray.constData()); + } + } +} + +IndexOutput* FSDirectory::createOutput(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QString file = directory + QDir::separator() + name; + if (QFileInfo(file).exists()) { + if (!QFile::remove(file)) { + QByteArray bArray("Cannot overwrite: "); + bArray.append(name.toLocal8Bit()); + _CLTHROWA(CL_ERR_IO, bArray.constData()); + } + } + return _CLNEW FSIndexOutput(file); +} + +LuceneLock* FSDirectory::makeLock(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + + QString lockFile(getLockPrefix()); + lockFile.append(QLatin1Char('-')).append(name); + + return _CLNEW FSLock(lockDir, lockFile); +} + +QString FSDirectory::toString() const +{ + return QString::fromLatin1("FSDirectory@").append(directory); +} + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/FSDirectory.h b/3rdparty/clucene/src/CLucene/store/FSDirectory.h new file mode 100644 index 000000000..e967380e0 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/FSDirectory.h @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_FSDirectory_ +#define _lucene_store_FSDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QFile> +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "Directory.h" +#include "Lock.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_DEF(store) + +/** +* Straightforward implementation of {@link Directory} as a directory of files. +* <p>If the system property 'disableLuceneLocks' has the String value of +* "true", lock creation will be disabled. +* +* @see Directory +*/ +class FSDirectory : public Directory +{ +public: + // Destructor - only call this if you are sure the directory + // is not being used anymore. Otherwise use the ref-counting + // facilities of _CLDECDELETE + ~FSDirectory(); + + // Get a list of strings, one for each file in the directory. + QStringList list() const; + + // Returns true iff a file with the given name exists. + bool fileExists(const QString& name) const; + + // Returns the text name of the directory + QString getDirName() const; ///<returns reference + + /** + Returns the directory instance for the named location. + + Do not delete this instance, only use close, otherwise other instances + will lose this instance. + + <p>Directories are cached, so that, for a given canonical path, the same + FSDirectory instance will always be returned. This permits + synchronization on directories. + + @param file the path to the directory. + @param create if true, create, or erase any existing contents. + @return the FSDirectory for the named file. + */ + static FSDirectory* getDirectory(const QString& file, const bool create); + + // Returns the time the named file was last modified. + int64_t fileModified(const QString& name) const; + + //static + // Returns the time the named file was last modified. + static int64_t fileModified(const QString& dir, const QString& name); + + // static + // Returns the length in bytes of a file in the directory. + int64_t fileLength(const QString& name) const; + + // Returns a stream reading an existing file. + IndexInput* openInput(const QString& name); + IndexInput* openInput(const QString& name, int32_t bufferSize); + + // Renames an existing file in the directory. + void renameFile(const QString& from, const QString& to); + + // Set the modified time of an existing file to now. + void touchFile(const QString& name); + + // Creates a new, empty file in the directory with the given name. + // Returns a stream writing this file. + IndexOutput* createOutput(const QString& name); + + // Construct a {@link Lock}. + // @param name the name of the lock file + LuceneLock* makeLock(const QString& name); + + // Decrease the ref-count to the directory by one. If the object is no + // longer needed, then the object is removed from the directory pool. + void close(); + + // If MMap is available, this can disable use of mmap reading. + void setUseMMap(bool value) { useMMap = value; } + + // Gets whether the directory is using MMap for inputstreams. + bool getUseMMap() const { return useMMap; } + + QString toString() const; + + static QString DirectoryType() { return QLatin1String("FS"); } + QString getDirectoryType() const { return QLatin1String("FS"); } + + // Set whether Lucene's use of lock files is disabled. By default, + // lock files are enabled. They should only be disabled if the index + // is on a read-only medium like a CD-ROM. + static void setDisableLocks(bool doDisableLocks) + { disableLocks = doDisableLocks; } + + // Returns whether Lucene's use of lock files is disabled. + // @return true if locks are disabled, false if locks are enabled. + static bool getDisableLocks() { return disableLocks; } + +protected: + FSDirectory(const QString& path, const bool createDir); + // Removes an existing file in the directory. + bool doDeleteFile(const QString& name); + +private: + class FSLock : public LuceneLock { + public: + FSLock (const QString& lockDir, const QString& name); + ~FSLock(); + + bool obtain(); + void release(); + bool isLocked(); + QString toString() const; + + QString lockDir; + QString lockFile; + }; + friend class FSDirectory::FSLock; + + class FSIndexInput : public BufferedIndexInput { + public: + FSIndexInput(const QString& path, int32_t bufferSize = + CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + ~FSIndexInput(); + + void close(); + IndexInput* clone() const; + + int64_t length() + { return handle->_length; } + + QString getDirectoryType() const + { return FSDirectory::DirectoryType(); } + + protected: + FSIndexInput(const FSIndexInput& clone); + // Random-access methods + void seekInternal(const int64_t position); + // IndexInput methods + void readInternal(uint8_t* b, const int32_t len); + + private: + // We used a shared handle between all the fsindexinput clones. + // This reduces number of file handles we need, and it means + // we dont have to use file tell (which is slow) before doing a read. + class SharedHandle : LUCENE_REFBASE { + public: + SharedHandle(); + ~SharedHandle(); + + int64_t _fpos; + int64_t _length; + + QFile fhandle; + DEFINE_MUTEX(*THIS_LOCK) + }; + SharedHandle* handle; + int64_t _pos; + }; + friend class FSDirectory::FSIndexInput; + + class FSIndexOutput : public BufferedIndexOutput { + public: + FSIndexOutput(const QString& path); + ~FSIndexOutput(); + + void close(); + int64_t length(); + void seek(const int64_t pos); + + protected: + void flushBuffer(const uint8_t* b, const int32_t size); + + private: + QFile fhandle; + }; + friend class FSDirectory::FSIndexOutput; + +private: + QString directory; + int refCount; + void create(); + + QString lockDir; + QString getLockPrefix() const; + static bool disableLocks; + + void priv_getFN(QString& buffer, const QString& name) const; + bool useMMap; +}; + +CL_NS_END + +#endif diff --git a/3rdparty/clucene/src/CLucene/store/IndexInput.cpp b/3rdparty/clucene/src/CLucene/store/IndexInput.cpp new file mode 100644 index 000000000..cf7bd16b7 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/IndexInput.cpp @@ -0,0 +1,233 @@ + /*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "IndexInput.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + + IndexInput::IndexInput() + { + } + IndexInput::IndexInput(const IndexInput& other) + { + } + + int32_t IndexInput::readInt() { + int32_t b = (readByte() << 24); + b |= (readByte() << 16); + b |= (readByte() << 8); + return (b | readByte()); + } + + int32_t IndexInput::readVInt() { + uint8_t b = readByte(); + int32_t i = b & 0x7F; + for (int32_t shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (b & 0x7F) << shift; + } + return i; + } + + int64_t IndexInput::readLong() { + int64_t i = ((int64_t)readInt() << 32); + return (i | ((int64_t)readInt() & 0xFFFFFFFFL)); + } + + int64_t IndexInput::readVLong() { + uint8_t b = readByte(); + int64_t i = b & 0x7F; + for (int32_t shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (((int64_t)b) & 0x7FL) << shift; + } + return i; + } + + void IndexInput::skipChars( const int32_t count) { + for (int32_t i = 0; i < count; i++) { + TCHAR b = readByte(); + if ((b & 0x80) == 0) { + // Do Nothing. + } else if ((b & 0xE0) != 0xE0) { + readByte(); + } else { + readByte(); + readByte(); + } + } + } + + int32_t IndexInput::readString(TCHAR* buffer, const int32_t maxLength){ + int32_t len = readVInt(); + int32_t ml=maxLength-1; + if ( len >= ml ){ + readChars(buffer, 0, ml); + buffer[ml] = 0; + //we have to finish reading all the data for this string! + if ( len-ml > 0 ){ + //seek(getFilePointer()+(len-ml)); <- that was the wrong way to "finish reading" + skipChars(len-ml); + } + return ml; + }else{ + readChars(buffer, 0, len); + buffer[len] = 0; + return len; + } + } + + TCHAR* IndexInput::readString(const bool _unique){ + int32_t len = readVInt(); + + if ( len == 0){ + if ( _unique ) //todo: does non unique ever occur? + return stringDuplicate(LUCENE_BLANK_STRING); + else + return LUCENE_BLANK_STRING; + } + + TCHAR* ret = _CL_NEWARRAY(TCHAR,len+1); + readChars(ret, 0, len); + ret[len] = 0; + + return ret; + } + + void IndexInput::readChars( TCHAR* buffer, const int32_t start, const int32_t len) { + const int32_t end = start + len; + TCHAR b; + for (int32_t i = start; i < end; ++i) { + b = readByte(); + if ((b & 0x80) == 0) { + b = (b & 0x7F); + } else if ((b & 0xE0) != 0xE0) { + b = (((b & 0x1F) << 6) + | (readByte() & 0x3F)); + } else { + b = ((b & 0x0F) << 12) | ((readByte() & 0x3F) << 6); + b |= (readByte() & 0x3F); + } + buffer[i] = b; + } + } + + + + + + +BufferedIndexInput::BufferedIndexInput(int32_t _bufferSize): + buffer(NULL), + bufferSize(_bufferSize), + bufferStart(0), + bufferLength(0), + bufferPosition(0) + { + } + + BufferedIndexInput::BufferedIndexInput(const BufferedIndexInput& other): + IndexInput(other), + buffer(NULL), + bufferSize(other.bufferSize), + bufferStart(other.bufferStart), + bufferLength(other.bufferLength), + bufferPosition(other.bufferPosition) + { + /* DSR: Does the fact that sometime clone.buffer is not NULL even when + ** clone.bufferLength is zero indicate memory corruption/leakage? + ** if ( clone.buffer != NULL) { */ + if (other.bufferLength != 0 && other.buffer != NULL) { + buffer = _CL_NEWARRAY(uint8_t,bufferLength); + memcpy(buffer,other.buffer,bufferLength * sizeof(uint8_t)); + } + } + + void BufferedIndexInput::readBytes(uint8_t* b, const int32_t len){ + if (len < bufferSize) { + for (int32_t i = 0; i < len; ++i) // read byte-by-byte + b[i] = readByte(); + } else { // read all-at-once + int64_t start = getFilePointer(); + seekInternal(start); + readInternal(b, len); + + bufferStart = start + len; // adjust stream variables + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read + } + } + + int64_t BufferedIndexInput::getFilePointer() const{ + return bufferStart + bufferPosition; + } + + void BufferedIndexInput::seek(const int64_t pos) { + if ( pos < 0 ) + _CLTHROWA(CL_ERR_IO, "IO Argument Error. Value must be a positive value."); + if (pos >= bufferStart && pos < (bufferStart + bufferLength)) + bufferPosition = (int32_t)(pos - bufferStart); // seek within buffer + else { + bufferStart = pos; + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read() + seekInternal(pos); + } + } + void BufferedIndexInput::close(){ + _CLDELETE_ARRAY(buffer); + bufferLength = 0; + bufferPosition = 0; + bufferStart = 0; + } + + + BufferedIndexInput::~BufferedIndexInput(){ + BufferedIndexInput::close(); + } + + void BufferedIndexInput::refill() { + int64_t start = bufferStart + bufferPosition; + int64_t end = start + bufferSize; + if (end > length()) // don't read past EOF + end = length(); + bufferLength = (int32_t)(end - start); + if (bufferLength == 0) + _CLTHROWA(CL_ERR_IO, "IndexInput read past EOF"); + + if (buffer == NULL){ + buffer = _CL_NEWARRAY(uint8_t,bufferSize); // allocate buffer lazily + } + readInternal(buffer, bufferLength); + + + bufferStart = start; + bufferPosition = 0; + } + + +IndexInputStream::IndexInputStream(IndexInput* input){ + this->input = input; + this->size = input->length(); + this->position = input->getFilePointer(); +} +IndexInputStream::~IndexInputStream(){ +} +int32_t IndexInputStream::fillBuffer(char* start, int32_t space){ + int64_t avail = input->length()-input->getFilePointer(); + if ( avail == 0 ) + return -1; + else if ( avail<space ) + space = (int32_t)avail; + + input->readBytes((uint8_t*)start,space); + return space; +} + +CL_NS_END + diff --git a/3rdparty/clucene/src/CLucene/store/IndexInput.h b/3rdparty/clucene/src/CLucene/store/IndexInput.h new file mode 100644 index 000000000..9453b5cf1 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/IndexInput.h @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_IndexInput_ +#define _lucene_store_IndexInput_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/util/bufferedstream.h" +#include "IndexOutput.h" + +CL_NS_DEF(store) + + /** Abstract base class for input from a file in a {@link Directory}. A + * random-access input stream. Used for all Lucene index input operations. + * @see Directory + * @see IndexOutput + */ + class IndexInput: LUCENE_BASE { + private: + void skipChars( const int32_t count); + protected: + IndexInput(); + IndexInput(const IndexInput& clone); + public: + virtual ~IndexInput(){} + virtual IndexInput* clone() const =0; + + DEFINE_MUTEX(THIS_LOCK) + + /** Reads and returns a single byte. + * @see IndexOutput#writeByte(byte) + */ + virtual uint8_t readByte() =0; + + /** Reads a specified number of bytes into an array at the specified offset. + * @param b the array to read bytes into + * @param offset the offset in the array to start storing bytes + * @param len the number of bytes to read + * @see IndexOutput#writeBytes(byte[],int32_t) + */ + virtual void readBytes(uint8_t* b, const int32_t len) =0; + + /** Reads four bytes and returns an int. + * @see IndexOutput#writeInt(int32_t) + */ + int32_t readInt(); + + /** Reads an int stored in variable-length format. Reads between one and + * five bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexOutput#writeVInt(int32_t) + */ + virtual int32_t readVInt(); + + /** Reads eight bytes and returns a long. + * @see IndexOutput#writeLong(long) + */ + int64_t readLong(); + + /** Reads a long stored in variable-length format. Reads between one and + * nine bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. */ + int64_t readVLong(); + + /** Reads a string. + * @see IndexOutput#writeString(String) + * maxLength is the amount read into the buffer, the whole string is still read from the stream + * returns the amount read + */ + int32_t readString(TCHAR* buffer, const int32_t maxlength); + + /** Reads a string. + * @see IndexOutput#writeString(String) + * If unique is true (default) the string will be duplicated. + * If false and the length is zero, LUCENE_BLANK_STRING is returned + */ + TCHAR* readString(const bool unique=true); + + + /** Reads UTF-8 encoded characters into an array. + * @param buffer the array to read characters into + * @param start the offset in the array to start storing characters + * @param length the number of characters to read + * @see IndexOutput#writeChars(String,int32_t,int32_t) + */ + void readChars( TCHAR* buffer, const int32_t start, const int32_t len); + + /** Closes the stream to futher operations. */ + virtual void close() =0; + + /** Returns the current position in this file, where the next read will + * occur. + * @see #seek(long) + */ + virtual int64_t getFilePointer() const =0; + + /** Sets current position in this file, where the next read will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos) =0; + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + virtual QString getDirectoryType() const = 0; + }; + + /** Abstract base class for input from a file in a {@link Directory}. A + * random-access input stream. Used for all Lucene index input operations. + * @see Directory + * @see IndexOutput + */ + class BufferedIndexInput: public IndexInput{ + private: + uint8_t* buffer; //array of bytes + void refill(); + protected: + int32_t bufferSize; //size of the buffer + int64_t bufferStart; // position in file of buffer + int32_t bufferLength; // end of valid l_byte_ts + int32_t bufferPosition; // next uint8_t to read + + /** Returns a clone of this stream. + * + * <p>Clones of a stream access the same data, and are positioned at the same + * point as the stream they were cloned from. + * + * <p>Expert: Subclasses must ensure that clones may be positioned at + * different points in the input from each other and from the stream they + * were cloned from. + */ + BufferedIndexInput(const BufferedIndexInput& clone); + BufferedIndexInput(int32_t bufferSize = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + public: + + virtual ~BufferedIndexInput(); + virtual IndexInput* clone() const = 0; + void close(); + inline uint8_t readByte(){ + if (bufferPosition >= bufferLength) + refill(); + + return buffer[bufferPosition++]; + } + void readBytes(uint8_t* b, const int32_t len); + int64_t getFilePointer() const; + void seek(const int64_t pos); + + protected: + /** Expert: implements buffer refill. Reads bytes from the current position + * in the input. + * @param b the array to read bytes into + * @param offset the offset in the array to start storing bytes + * @param length the number of bytes to read + */ + virtual void readInternal(uint8_t* b, const int32_t len) = 0; + + /** Expert: implements seek. Sets current position in this file, where the + * next {@link #readInternal(byte[],int32_t,int32_t)} will occur. + * @see #readInternal(byte[],int32_t,int32_t) + */ + virtual void seekInternal(const int64_t pos) = 0; + }; + + /** + * JStream InputStream which reads from an IndexInput. This class is + * used by the FieldReader to create binary fields. You can then use + * a GZipInputStream to read compressed data or any of the other + * JStream stream types. + * + */ + class IndexInputStream: public jstreams::BufferedInputStream<char>{ + IndexInput* input; + public: + IndexInputStream(IndexInput* input); + ~IndexInputStream(); + int32_t fillBuffer(char* start, int32_t space); + }; +CL_NS_END +#endif diff --git a/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp b/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp new file mode 100644 index 000000000..04f78c348 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp @@ -0,0 +1,163 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "IndexOutput.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + + + IndexOutput::IndexOutput() + { + } + + IndexOutput::~IndexOutput(){ + } + + BufferedIndexOutput::BufferedIndexOutput() + { + buffer = _CL_NEWARRAY(uint8_t, BUFFER_SIZE ); + bufferStart = 0; + bufferPosition = 0; + } + + BufferedIndexOutput::~BufferedIndexOutput(){ + if ( buffer != NULL ) + close(); + } + + void BufferedIndexOutput::close(){ + flush(); + _CLDELETE_ARRAY( buffer ); + + bufferStart = 0; + bufferPosition = 0; + } + + void BufferedIndexOutput::writeByte(const uint8_t b) { + CND_PRECONDITION(buffer!=NULL,"IndexOutput is closed") + if (bufferPosition >= BUFFER_SIZE) + flush(); + buffer[bufferPosition++] = b; + } + + void BufferedIndexOutput::writeBytes(const uint8_t* b, const int32_t length) { + if ( length < 0 ) + _CLTHROWA(CL_ERR_IllegalArgument, "IO Argument Error. Value must be a positive value."); + int32_t bytesLeft = BUFFER_SIZE - bufferPosition; + // is there enough space in the buffer? + if (bytesLeft >= length) { + // we add the data to the end of the buffer + memcpy(buffer + bufferPosition, b, length); + bufferPosition += length; + // if the buffer is full, flush it + if (BUFFER_SIZE - bufferPosition == 0) + flush(); + } else { + // is data larger then buffer? + if (length > BUFFER_SIZE) { + // we flush the buffer + if (bufferPosition > 0) + flush(); + // and write data at once + flushBuffer(b, length); + bufferStart += length; + } else { + // we fill/flush the buffer (until the input is written) + int64_t pos = 0; // position in the input data + int32_t pieceLength; + while (pos < length) { + if ( length - pos < bytesLeft ) + pieceLength = length - pos; + else + pieceLength = bytesLeft; + memcpy(buffer + bufferPosition, b + pos, pieceLength); + pos += pieceLength; + bufferPosition += pieceLength; + // if the buffer is full, flush it + bytesLeft = BUFFER_SIZE - bufferPosition; + if (bytesLeft == 0) { + flush(); + bytesLeft = BUFFER_SIZE; + } + } + } + } + } + + void IndexOutput::writeInt(const int32_t i) { + writeByte((uint8_t)(i >> 24)); + writeByte((uint8_t)(i >> 16)); + writeByte((uint8_t)(i >> 8)); + writeByte((uint8_t) i); + } + + void IndexOutput::writeVInt(const int32_t vi) { + uint32_t i = vi; + while ((i & ~0x7F) != 0) { + writeByte((uint8_t)((i & 0x7f) | 0x80)); + i >>= 7; //doing unsigned shift + } + writeByte( (uint8_t)i ); + } + + void IndexOutput::writeLong(const int64_t i) { + writeInt((int32_t) (i >> 32)); + writeInt((int32_t) i); + } + + void IndexOutput::writeVLong(const int64_t vi) { + uint64_t i = vi; + while ((i & ~0x7F) != 0) { + writeByte((uint8_t)((i & 0x7f) | 0x80)); + i >>= 7; //doing unsigned shift + } + writeByte((uint8_t)i); + } + + void IndexOutput::writeString(const TCHAR* s, const int32_t length ) { + writeVInt(length); + writeChars(s, 0, length); + } + + void IndexOutput::writeChars(const TCHAR* s, const int32_t start, const int32_t length){ + if ( length < 0 || start < 0 ) + _CLTHROWA(CL_ERR_IllegalArgument, "IO Argument Error. Value must be a positive value."); + + const int32_t end = start + length; + for (int32_t i = start; i < end; ++i) { + const int32_t code = (int32_t)s[i]; + if (code >= 0x01 && code <= 0x7F) + writeByte((uint8_t)code); + else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) { + writeByte((uint8_t)(0xC0 | (code >> 6))); + writeByte((uint8_t)(0x80 | (code & 0x3F))); + } else { + writeByte((uint8_t)(0xE0 | (((uint32_t)code) >> 12))); //unsigned shift + writeByte((uint8_t)(0x80 | ((code >> 6) & 0x3F))); + writeByte((uint8_t)(0x80 | (code & 0x3F))); + } + } + } + + + int64_t BufferedIndexOutput::getFilePointer() const{ + return bufferStart + bufferPosition; + } + + void BufferedIndexOutput::seek(const int64_t pos) { + flush(); + bufferStart = pos; + } + + void BufferedIndexOutput::flush() { + flushBuffer(buffer, bufferPosition); + bufferStart += bufferPosition; + bufferPosition = 0; + } + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/IndexOutput.h b/3rdparty/clucene/src/CLucene/store/IndexOutput.h new file mode 100644 index 000000000..c47ee73a7 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/IndexOutput.h @@ -0,0 +1,152 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_IndexOutput_ +#define _lucene_store_IndexOutput_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + + +/** Abstract class for output to a file in a Directory. A random-access output +* stream. Used for all Lucene index output operations. +* @see Directory +* @see IndexInput +*/ +class IndexOutput:LUCENE_BASE{ + bool isclosed; +public: + IndexOutput(); + virtual ~IndexOutput(); + + /** Writes a single byte. + * @see IndexInput#readByte() + */ + virtual void writeByte(const uint8_t b) = 0; + + /** Writes an array of bytes. + * @param b the bytes to write + * @param length the number of bytes to write + * @see IndexInput#readBytes(byte[],int32_t,int32_t) + */ + virtual void writeBytes(const uint8_t* b, const int32_t length) = 0; + + /** Writes an int as four bytes. + * @see IndexInput#readInt() + */ + void writeInt(const int32_t i); + + /** Writes an int in a variable-length format. Writes between one and + * five bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexInput#readVInt() + */ + void writeVInt(const int32_t vi); + + /** Writes a long as eight bytes. + * @see IndexInput#readLong() + */ + void writeLong(const int64_t i); + + /** Writes an long in a variable-length format. Writes between one and five + * bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexInput#readVLong() + */ + void writeVLong(const int64_t vi); + + /** Writes a string. + * @see IndexInput#readString() + */ + void writeString(const TCHAR* s, const int32_t length); + + /** Writes a sequence of UTF-8 encoded characters from a string. + * @param s the source of the characters + * @param start the first character in the sequence + * @param length the number of characters in the sequence + * @see IndexInput#readChars(char[],int32_t,int32_t) + */ + void writeChars(const TCHAR* s, const int32_t start, const int32_t length); + + /** Closes this stream to further operations. */ + virtual void close() = 0; + + /** Returns the current position in this file, where the next write will + * occur. + * @see #seek(long) + */ + virtual int64_t getFilePointer() const = 0; + + /** Sets current position in this file, where the next write will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos) = 0; + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + /** Forces any buffered output to be written. */ + virtual void flush() = 0; +}; + +/** Base implementation class for buffered {@link IndexOutput}. */ +class BufferedIndexOutput : public IndexOutput{ +public: + LUCENE_STATIC_CONSTANT(int32_t, BUFFER_SIZE=LUCENE_STREAM_BUFFER_SIZE); +private: + uint8_t* buffer; + int64_t bufferStart; // position in file of buffer + int32_t bufferPosition; // position in buffer + +public: + BufferedIndexOutput(); + virtual ~BufferedIndexOutput(); + + /** Writes a single byte. + * @see IndexInput#readByte() + */ + virtual void writeByte(const uint8_t b); + + /** Writes an array of bytes. + * @param b the bytes to write + * @param length the number of bytes to write + * @see IndexInput#readBytes(byte[],int32_t,int32_t) + */ + virtual void writeBytes(const uint8_t* b, const int32_t length); + + /** Closes this stream to further operations. */ + virtual void close(); + + /** Returns the current position in this file, where the next write will + * occur. + * @see #seek(long) + */ + int64_t getFilePointer() const; + + /** Sets current position in this file, where the next write will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos); + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + /** Forces any buffered output to be written. */ + void flush(); + +protected: + /** Expert: implements buffer write. Writes bytes at the current position in + * the output. + * @param b the bytes to write + * @param len the number of bytes to write + */ + virtual void flushBuffer(const uint8_t* b, const int32_t len) = 0; +}; + +CL_NS_END +#endif diff --git a/3rdparty/clucene/src/CLucene/store/InputStream.h b/3rdparty/clucene/src/CLucene/store/InputStream.h new file mode 100644 index 000000000..f56819eeb --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/InputStream.h @@ -0,0 +1,21 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_InputStream_ +#define _lucene_store_InputStream_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + +deprecated... please use IndexInput.h header +and change InputStream to IndexInput + + +CL_NS_END +#endif diff --git a/3rdparty/clucene/src/CLucene/store/Lock.cpp b/3rdparty/clucene/src/CLucene/store/Lock.cpp new file mode 100644 index 000000000..a66e784b0 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/Lock.cpp @@ -0,0 +1,27 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Lock.h" + +CL_NS_DEF(store) + + bool LuceneLock::obtain(int64_t lockWaitTimeout) { + bool locked = obtain(); + int maxSleepCount = (int)(lockWaitTimeout / LOCK_POLL_INTERVAL); + int sleepCount = 0; + while (!locked) { + if (sleepCount++ == maxSleepCount) { + _CLTHROWA(CL_ERR_IO,"Lock obtain timed out"); + } + _LUCENE_SLEEP(LOCK_POLL_INTERVAL); + locked = obtain(); + } + return locked; + } + + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/Lock.h b/3rdparty/clucene/src/CLucene/store/Lock.h new file mode 100644 index 000000000..b5dda3b06 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/Lock.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_Lock_ +#define _lucene_store_Lock_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + +class LuceneLock : LUCENE_BASE +{ +public: + LUCENE_STATIC_CONSTANT(int64_t, LOCK_POLL_INTERVAL = 1000); + + virtual ~LuceneLock() {} + + // Attempts to obtain exclusive access and immediately return upon success + // or failure. Return true if exclusive access is obtained. + virtual bool obtain() = 0; + + // Attempts to obtain an exclusive lock within amount of time given. + // Currently polls once per second until lockWaitTimeout is passed. + // @param lockWaitTimeout length of time to wait in ms + // @return true if lock was obtained + // @throws IOException if lock wait times out or obtain() throws an IOException + bool obtain(int64_t lockWaitTimeout); + + // Release exclusive access. + virtual void release() = 0; + + // Returns true if the resource is currently locked. Note that one must + // still call {@link #obtain()} before using the resource. + virtual bool isLocked() = 0; + + virtual QString toString() const = 0; +}; + + +// Utility class for executing code with exclusive access. +template<typename T> +class LuceneLockWith +{ +public: + // Constructs an executor that will grab the named lock. Defaults + // lockWaitTimeout to LUCENE_COMMIT_LOCK_TIMEOUT. + // @deprecated Kept only to avoid breaking existing code. + LuceneLockWith(LuceneLock* lock, int64_t lockWaitTimeout) + { + this->lock = lock; + this->lockWaitTimeout = lockWaitTimeout; + } + + virtual ~LuceneLockWith() {} + + // Calls {@link #doBody} while <i>lock</i> is obtained. Blocks if lock + // cannot be obtained immediately. Retries to obtain lock once per second + // until it is obtained, or until it has tried ten times. Lock is released + // when {@link #doBody} exits. + T runAndReturn() + { + bool locked = false; + T ret = NULL; + try { + locked = lock->obtain(lockWaitTimeout); + ret = doBody(); + } _CLFINALLY ( + if (locked) + lock->release(); + ); + return ret; + } + + // @see runAndReturn + // Same as runAndReturn, except doesn't return any value. The only + // difference is that no void values are used + void run() + { + bool locked = false; + try { + locked = lock->obtain(lockWaitTimeout); + doBody(); + } _CLFINALLY ( + if (locked) + lock->release(); + ); + } + +protected: + virtual T doBody() = 0; + +private: + LuceneLock* lock; + int64_t lockWaitTimeout; +}; + +CL_NS_END + +#endif diff --git a/3rdparty/clucene/src/CLucene/store/MMapInput.cpp b/3rdparty/clucene/src/CLucene/store/MMapInput.cpp new file mode 100644 index 000000000..d660032c6 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/MMapInput.cpp @@ -0,0 +1,203 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#if defined(LUCENE_FS_MMAP) + +#include "FSDirectory.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +#ifndef _CLCOMPILER_MSVC + #include <sys/mman.h> +#endif + +CL_NS_DEF(store) +CL_NS_USE(util) + + FSDirectory::MMapIndexInput::MMapIndexInput(const char* path): + pos(0), + data(NULL), + _length(0), + isClone(false) + { + //Func - Constructor. + // Opens the file named path + //Pre - path != NULL + //Post - if the file could not be opened an exception is thrown. + + CND_PRECONDITION(path != NULL, "path is NULL"); + +#ifdef _CLCOMPILER_MSVC + mmaphandle = NULL; + fhandle = CreateFileA(path,GENERIC_READ,FILE_SHARE_READ, 0,OPEN_EXISTING,0,0); + + //Check if a valid fhandle was retrieved + if (fhandle < 0){ + DWORD err = GetLastError(); + if ( err == ERROR_FILE_NOT_FOUND ) + _CLTHROWA(CL_ERR_IO, "File does not exist"); + else if ( err == EACCES ) + _CLTHROWA(ERROR_ACCESS_DENIED, "File Access denied"); + else if ( err == ERROR_TOO_MANY_OPEN_FILES ) + _CLTHROWA(CL_ERR_IO, "Too many open files"); + else + _CLTHROWA(CL_ERR_IO, "File IO Error"); + } + + DWORD dummy=0; + _length = GetFileSize(fhandle,&dummy); + + if ( _length > 0 ){ + mmaphandle = CreateFileMappingA(fhandle,NULL,PAGE_READONLY,0,0,NULL); + if ( mmaphandle != NULL ){ + void* address = MapViewOfFile(mmaphandle,FILE_MAP_READ,0,0,0); + if ( address != NULL ){ + data = (uint8_t*)address; + return; //SUCCESS! + } + } + CloseHandle(mmaphandle); + + char* lpMsgBuf=0; + DWORD dw = GetLastError(); + + FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + dw, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + lpMsgBuf, + 0, NULL ); + + char* errstr = _CL_NEWARRAY(char, strlen(lpMsgBuf)+40); + sprintf(errstr, "MMapIndexInput::MMapIndexInput failed with error %d: %s", dw, lpMsgBuf); + LocalFree(lpMsgBuf); + + _CLTHROWA_DEL(CL_ERR_IO,errstr); + } + +#else //_CLCOMPILER_MSVC + fhandle = ::open (path, O_RDONLY); + if (fhandle < 0){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + // stat it + struct stat sb; + if (::fstat (fhandle, &sb)){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + // get length from stat + _length = sb.st_size; + + // mmap the file + void* address = ::mmap(0, _length, PROT_READ, MAP_SHARED, fhandle, 0); + if (address == MAP_FAILED){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + data = (uint8_t*)address; + } + } + } +#endif + } + + FSDirectory::MMapIndexInput::MMapIndexInput(const MMapIndexInput& clone): IndexInput(clone){ + //Func - Constructor + // Uses clone for its initialization + //Pre - clone is a valide instance of FSIndexInput + //Post - The instance has been created and initialized by clone + +#ifdef _CLCOMPILER_MSVC + mmaphandle = NULL; + fhandle = NULL; +#endif + + data = clone.data; + pos = clone.pos; + + //clone the file length + _length = clone._length; + //Keep in mind that this instance is a clone + isClone = true; + } + + uint8_t FSDirectory::MMapIndexInput::readByte(){ + return *(data+(pos++)); + } + + void FSDirectory::MMapIndexInput::readBytes(uint8_t* b, const int32_t len){ + memcpy(b, data+pos, len); + pos+=len; + } + int32_t FSDirectory::MMapIndexInput::readVInt(){ + uint8_t b = *(data+(pos++)); + int32_t i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = *(data+(pos++)); + i |= (b & 0x7F) << shift; + } + return i; + } + int64_t FSDirectory::MMapIndexInput::getFilePointer() const{ + return pos; + } + void FSDirectory::MMapIndexInput::seek(const int64_t pos){ + this->pos=pos; + } + + FSDirectory::MMapIndexInput::~MMapIndexInput(){ + //Func - Destructor + //Pre - True + //Post - The file for which this instance is responsible has been closed. + // The instance has been destroyed + + close(); + } + + IndexInput* FSDirectory::MMapIndexInput::clone() const + { + return _CLNEW FSDirectory::MMapIndexInput(*this); + } + void FSDirectory::MMapIndexInput::close() { + //IndexInput::close(); + + if ( !isClone ){ +#ifdef _CLCOMPILER_MSVC + if ( data != NULL ){ + if ( ! UnmapViewOfFile(data) ){ + CND_PRECONDITION( false, "UnmapViewOfFile(data) failed"); //todo: change to rich error + } + } + + if ( mmaphandle != NULL ){ + if ( ! CloseHandle(mmaphandle) ){ + CND_PRECONDITION( false, "CloseHandle(mmaphandle) failed"); + } + } + if ( fhandle != NULL ){ + if ( !CloseHandle(fhandle) ){ + CND_PRECONDITION( false, "CloseHandle(fhandle) failed"); + } + } + mmaphandle = NULL; + fhandle = NULL; +#else + if ( data != NULL ) + ::munmap(data, _length); + if ( fhandle > 0 ) + ::close(fhandle); + fhandle = 0; +#endif + } + data = NULL; + pos = 0; + } + + +CL_NS_END +#endif diff --git a/3rdparty/clucene/src/CLucene/store/OutputStream.h b/3rdparty/clucene/src/CLucene/store/OutputStream.h new file mode 100644 index 000000000..a82d6718a --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/OutputStream.h @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_IndexOutput_ +#define _lucene_store_IndexOutput_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + + +deprecated... please use IndexOutput.h header +and change OutputStream to OutdexInput + + + +CL_NS_END + +#endif // _lucene_store_IndexOutput_ diff --git a/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp b/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp new file mode 100644 index 000000000..b0a7c4d64 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp @@ -0,0 +1,446 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#include "CLucene/StdHeader.h" +#include "RAMDirectory.h" + +#include "Lock.h" +#include "Directory.h" +#include "FSDirectory.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + +RAMFile::RAMFile() +{ + length = 0; + lastModified = Misc::currentTimeMillis(); +} + +RAMFile::~RAMFile() +{ +} + + +RAMDirectory::RAMLock::RAMLock(const QString& name, RAMDirectory* dir) + : directory(dir) +{ + fname = name; +} + +RAMDirectory::RAMLock::~RAMLock() +{ + directory = NULL; +} + +QString RAMDirectory::RAMLock::toString() const +{ + return QLatin1String("LockFile@RAM"); +} + +bool RAMDirectory::RAMLock::isLocked() +{ + return directory->fileExists(fname); +} + +bool RAMDirectory::RAMLock::obtain() +{ + SCOPED_LOCK_MUTEX(directory->files_mutex); + if (!directory->fileExists(fname)) { + IndexOutput* tmp = directory->createOutput(fname); + tmp->close(); + _CLDELETE(tmp); + + return true; + } + return false; +} + +void RAMDirectory::RAMLock::release() +{ + directory->deleteFile(fname); +} + +RAMIndexOutput::~RAMIndexOutput() +{ + if (deleteFile) + _CLDELETE(file); + file = NULL; +} + +RAMIndexOutput::RAMIndexOutput(RAMFile* f) + : file(f) +{ + pointer = 0; + deleteFile = false; +} + +RAMIndexOutput::RAMIndexOutput() + : file(_CLNEW RAMFile) +{ + pointer = 0; + deleteFile = true; +} + +void RAMIndexOutput::writeTo(IndexOutput* out) +{ + flush(); + int64_t end = file->length; + int64_t pos = 0; + int32_t p = 0; + while (pos < end) { + int32_t length = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int64_t nextPos = pos + length; + if (nextPos > end) { // at the last buffer + length = (int32_t)(end - pos); + } + out->writeBytes((uint8_t*)file->buffers[p++], length); + pos = nextPos; + } +} + +void RAMIndexOutput::reset() +{ + seek(_ILONGLONG(0)); + file->length = _ILONGLONG(0); +} + +void RAMIndexOutput::flushBuffer(const uint8_t* src, const int32_t len) +{ + uint8_t* b = NULL; + int32_t bufferPos = 0; + while (bufferPos != len) { + uint32_t bufferNumber = pointer/CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bufferOffset = pointer%CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bytesInBuffer = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE - bufferOffset; + int32_t remainInSrcBuffer = len - bufferPos; + int32_t bytesToCopy = bytesInBuffer >= remainInSrcBuffer ? remainInSrcBuffer : bytesInBuffer; + + if (bufferNumber == file->buffers.size()){ + b = _CL_NEWARRAY(uint8_t, CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + file->buffers.push_back( b ); + }else{ + b = file->buffers[bufferNumber]; + } + memcpy(b+bufferOffset, src+bufferPos, bytesToCopy * sizeof(uint8_t)); + bufferPos += bytesToCopy; + pointer += bytesToCopy; + } + if (pointer > file->length) + file->length = pointer; + + file->lastModified = Misc::currentTimeMillis(); +} + +void RAMIndexOutput::close() +{ + BufferedIndexOutput::close(); +} + +/** Random-at methods */ +void RAMIndexOutput::seek(const int64_t pos) +{ + BufferedIndexOutput::seek(pos); + pointer = (int32_t)pos; +} + +int64_t RAMIndexOutput::length() +{ + return file->length; +} + + +RAMIndexInput::RAMIndexInput(RAMFile* f) + : file(f) +{ + pointer = 0; + _length = f->length; +} + +RAMIndexInput::RAMIndexInput(const RAMIndexInput& other) + : BufferedIndexInput(other) +{ + file = other.file; + pointer = other.pointer; + _length = other._length; +} + +RAMIndexInput::~RAMIndexInput() +{ + RAMIndexInput::close(); +} + +IndexInput* RAMIndexInput::clone() const +{ + return _CLNEW RAMIndexInput(*this); +} + +int64_t RAMIndexInput::length() +{ + return _length; +} + +QString RAMIndexInput::getDirectoryType() const +{ + return RAMDirectory::DirectoryType(); +} + +void RAMIndexInput::readInternal(uint8_t* dest, const int32_t len) +{ + const int64_t bytesAvailable = file->length - pointer; + int64_t remainder = len <= bytesAvailable ? len : bytesAvailable; + int32_t start = pointer; + int32_t destOffset = 0; + while (remainder != 0) { + int32_t bufferNumber = start / CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bufferOffset = start % CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bytesInBuffer = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE - bufferOffset; + + /* The buffer's entire length (bufferLength) is defined by IndexInput.h + ** as int32_t, so obviously the number of bytes in a given segment of the + ** buffer won't exceed the the capacity of int32_t. Therefore, the + ** int64_t->int32_t cast on the next line is safe. */ + int32_t bytesToCopy = bytesInBuffer >= remainder ? static_cast<int32_t>(remainder) : bytesInBuffer; + uint8_t* b = file->buffers[bufferNumber]; + memcpy(dest + destOffset, b + bufferOffset, bytesToCopy * sizeof(uint8_t)); + + destOffset += bytesToCopy; + start += bytesToCopy; + remainder -= bytesToCopy; + pointer += bytesToCopy; + } +} + +void RAMIndexInput::close() +{ + BufferedIndexInput::close(); +} + +void RAMIndexInput::seekInternal(const int64_t pos) +{ + CND_PRECONDITION(pos >= 0 && pos < this->_length, "Seeking out of range") + pointer = (int32_t)pos; +} + +// #pragma mark -- RAMDirectory + +QStringList RAMDirectory::list() const +{ + SCOPED_LOCK_MUTEX(files_mutex); + + QStringList names; + + FileMap::const_iterator itr; + for (itr = files.begin(); itr != files.end(); ++itr) + names.push_back(itr->first); + + return names; +} + +RAMDirectory::RAMDirectory() + : Directory() + , files(false, true) +{ +} + +RAMDirectory::~RAMDirectory() +{ + //todo: should call close directory? +} + +void RAMDirectory::_copyFromDir(Directory* dir, bool closeDir) +{ + QStringList names = dir->list(); + uint8_t buf[CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE]; + + foreach (const QString& name, names) { + if (!CL_NS(index)::IndexReader::isLuceneFile(name)) + continue; + + // make place on ram disk + IndexOutput* os = createOutput(name); + // read current file + IndexInput* is = dir->openInput(name); + + // and copy to ram disk + //todo: this could be a problem when copying from big indexes... + int64_t readCount = 0; + int64_t len = is->length(); + while (readCount < len) { + int32_t toRead = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + if ((readCount + toRead) > len) + toRead = int32_t(len - readCount); + is->readBytes(buf, toRead); + os->writeBytes(buf, toRead); + readCount += toRead; + } + + // graceful cleanup + is->close(); + _CLDELETE(is); + os->close(); + _CLDELETE(os); + } + if (closeDir) + dir->close(); +} + +RAMDirectory::RAMDirectory(Directory* dir) + : Directory() + , files(false, true) +{ + _copyFromDir(dir, false); +} + +RAMDirectory::RAMDirectory(const QString& dir) + : Directory() + , files(false, true) +{ + Directory* fsdir = FSDirectory::getDirectory(dir, false); + try { + _copyFromDir(fsdir, false); + } _CLFINALLY ( + fsdir->close(); + _CLDECDELETE(fsdir); + ); +} + +bool RAMDirectory::fileExists(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + return files.exists(name); +} + +int64_t RAMDirectory::fileModified(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + const RAMFile* f = files.get(name); + return f->lastModified; +} + +int64_t RAMDirectory::fileLength(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + RAMFile* f = files.get(name); + return f->length; +} + + +IndexInput* RAMDirectory::openInput(const QString& name) +{ + SCOPED_LOCK_MUTEX(files_mutex); + RAMFile* file = files.get(name); + if (file == NULL) { + _CLTHROWA(CL_ERR_IO, // DSR:PROPOSED: Better error checking. + "[RAMDirectory::open] The requested file does not exist."); + } + return _CLNEW RAMIndexInput(file); +} + +void RAMDirectory::close() +{ + SCOPED_LOCK_MUTEX(files_mutex); + files.clear(); +} + +bool RAMDirectory::doDeleteFile(const QString& name) +{ + SCOPED_LOCK_MUTEX(files_mutex); + files.remove(name); + return true; +} + +void RAMDirectory::renameFile(const QString& from, const QString& to) +{ + SCOPED_LOCK_MUTEX(files_mutex); + FileMap::iterator itr = files.find(from); + + /* DSR:CL_BUG_LEAK: + ** If a file named $to already existed, its old value was leaked. + ** My inclination would be to prevent this implicit deletion with an + ** exception, but it happens routinely in CLucene's internals (e.g., during + ** IndexWriter.addIndexes with the file named 'segments'). */ + if (files.exists(to)) + files.remove(to); + + if (itr == files.end()) { + char tmp[1024]; + _snprintf(tmp, 1024, "cannot rename %s, file does not exist", + from.toLocal8Bit().constData()); + _CLTHROWT(CL_ERR_IO, tmp); + } + + CND_PRECONDITION(itr != files.end(), "itr == files.end()") + + RAMFile* file = itr->second; + files.removeitr(itr, true, true); + files.put(to, file); +} + + +void RAMDirectory::touchFile(const QString& name) +{ + RAMFile* file = NULL; + { + SCOPED_LOCK_MUTEX(files_mutex); + file = files.get(name); + } + uint64_t ts1 = file->lastModified; + uint64_t ts2 = Misc::currentTimeMillis(); + + //make sure that the time has actually changed + while (ts1 == ts2) { + _LUCENE_SLEEP(1); + ts2 = Misc::currentTimeMillis(); + }; + + file->lastModified = ts2; +} + +IndexOutput* RAMDirectory::createOutput(const QString& name) +{ + /* Check the $files VoidMap to see if there was a previous file named + ** $name. If so, delete the old RAMFile object, but reuse the existing + ** char buffer ($n) that holds the filename. If not, duplicate the + ** supplied filename buffer ($name) and pass ownership of that memory ($n) + ** to $files. */ + + SCOPED_LOCK_MUTEX(files_mutex); + + QString n = files.getKey(name); + if (!n.isEmpty()) { + RAMFile* rf = files.get(name); + _CLDELETE(rf); + } else { + n = name; + } + + RAMFile* file = _CLNEW RAMFile(); +#ifdef _DEBUG + file->filename = n; +#endif + files[n] = file; + + return _CLNEW RAMIndexOutput(file); +} + +LuceneLock* RAMDirectory::makeLock(const QString& name) +{ + return _CLNEW RAMLock(name, this); +} + +QString RAMDirectory::toString() const +{ + return QLatin1String("RAMDirectory"); +} + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/RAMDirectory.h b/3rdparty/clucene/src/CLucene/store/RAMDirectory.h new file mode 100644 index 000000000..af92e30b2 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/RAMDirectory.h @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_RAMDirectory_ +#define _lucene_store_RAMDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "Lock.h" +#include "Directory.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Arrays.h" + +CL_NS_DEF(store) + +class RAMFile : LUCENE_BASE +{ +public: + CL_NS(util)::CLVector<uint8_t*, CL_NS(util)::Deletor::Array<uint8_t> > buffers; + int64_t length; + uint64_t lastModified; + +#ifdef _DEBUG + QString filename; +#endif + + RAMFile(); + ~RAMFile(); +}; + +class RAMIndexOutput : public BufferedIndexOutput +{ +protected: + RAMFile* file; + int32_t pointer; + bool deleteFile; + + // output methods: + void flushBuffer(const uint8_t* src, const int32_t len); + +public: + RAMIndexOutput(RAMFile* f); + RAMIndexOutput(); + /** Construct an empty output buffer. */ + virtual ~RAMIndexOutput(); + + virtual void close(); + + // Random-at methods + virtual void seek(const int64_t pos); + int64_t length(); + /** Resets this to an empty buffer. */ + void reset(); + /** Copy the current contents of this buffer to the named output. */ + void writeTo(IndexOutput* output); +}; + +class RAMIndexInput : public BufferedIndexInput +{ +private: + RAMFile* file; + int32_t pointer; + int64_t _length; + +protected: + /** IndexInput methods */ + RAMIndexInput(const RAMIndexInput& clone); + void readInternal(uint8_t *dest, const int32_t len); + + /** Random-at methods */ + void seekInternal(const int64_t pos); + +public: + RAMIndexInput(RAMFile* f); + ~RAMIndexInput(); + IndexInput* clone() const; + + void close(); + int64_t length(); + QString getDirectoryType() const; +}; + + +/** +* A memory-resident {@link Directory} implementation. +*/ +class RAMDirectory : public Directory +{ + class RAMLock : public LuceneLock + { + private: + RAMDirectory* directory; + QString fname; + public: + RAMLock(const QString& name, RAMDirectory* dir); + virtual ~RAMLock(); + bool obtain(); + void release(); + bool isLocked(); + virtual QString toString() const; + }; + + typedef CL_NS(util)::CLHashMap<QString, RAMFile*, + CL_NS(util)::Compare::Qstring, CL_NS(util)::Equals::Qstring, + CL_NS(util)::Deletor::DummyQString, + CL_NS(util)::Deletor::Object<RAMFile> > FileMap; + +protected: + /// Removes an existing file in the directory. + virtual bool doDeleteFile(const QString& name); + + /** + * Creates a new <code>RAMDirectory</code> instance from a different + * <code>Directory</code> implementation. This can be used to load + * a disk-based index into memory. + * <P> + * This should be used only with indices that can fit into memory. + * + * @param dir a <code>Directory</code> value + * @exception IOException if an error occurs + */ + void _copyFromDir(Directory* dir, bool closeDir); + FileMap files; // unlike the java Hashtable, FileMap is not synchronized, and all access must be protected by a lock + +public: +#ifndef _CL_DISABLE_MULTITHREADING //do this so that the mutable keyword still works without mt enabled + mutable DEFINE_MUTEX(files_mutex) // mutable: const methods must also be able to synchronize properly +#endif + + // Returns a null terminated array of strings, one for each file in the directory. + QStringList list() const; + + /** Constructs an empty {@link Directory}. */ + RAMDirectory(); + + // Destructor - only call this if you are sure the directory + // is not being used anymore. Otherwise use the ref-counting + // facilities of dir->close + virtual ~RAMDirectory(); + RAMDirectory(Directory* dir); + + /** + * Creates a new <code>RAMDirectory</code> instance from the {@link FSDirectory}. + * + * @param dir a <code>String</code> specifying the full index directory path + */ + RAMDirectory(const QString& dir); + + /// Returns true iff the named file exists in this directory. + bool fileExists(const QString& name) const; + + /// Returns the time the named file was last modified. + int64_t fileModified(const QString& name) const; + + /// Returns the length in bytes of a file in the directory. + int64_t fileLength(const QString& name) const; + + /// Removes an existing file in the directory. + virtual void renameFile(const QString& from, const QString& to); + + /** Set the modified time of an existing file to now. */ + void touchFile(const QString& name); + + /// Creates a new, empty file in the directory with the given name. + /// Returns a stream writing this file. + virtual IndexOutput* createOutput(const QString& name); + + /// Construct a {@link Lock}. + /// @param name the name of the lock file + LuceneLock* makeLock(const QString& name); + + /// Returns a stream reading an existing file. + IndexInput* openInput(const QString& name); + + virtual void close(); + + QString toString() const; + + static QString DirectoryType() { return QLatin1String("RAM"); } + QString getDirectoryType() const { return DirectoryType(); } +}; + +CL_NS_END + +#endif diff --git a/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp b/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp new file mode 100644 index 000000000..056fa9bc3 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "TransactionalRAMDirectory.h" + +CL_NS_DEF(store) +CL_NS_USE(util) + +TransactionalRAMDirectory::TransactionalRAMDirectory() + : RAMDirectory() + , filesToRestoreOnAbort(false, true) +{ + transOpen = false; +} + +TransactionalRAMDirectory::~TransactionalRAMDirectory() +{ +} + +bool TransactionalRAMDirectory::archiveOrigFileIfNecessary(const QString& name) +{ + // If a file named $name was present when the transaction started and the + // original RAMFile object has not been archived for restoration upon + // transaction abort, then do so, and return true. + // In any other case, return false. + if (fileExists(name) && filesToRemoveOnAbort.find(name) == filesToRemoveOnAbort.end()) { + // The file exists, but isn't recorded as having been created after the + // start of the transaction, so it must've been present at the start of + // the transaction. + + // Transfer memory ownership of both the key and the value from files to + // filesToRestoreOnAbort. + QString origName = files.getKey(name); + RAMFile* origFile = files.get(name); + files.remove(name, true, true); + filesToRestoreOnAbort.put(origName, origFile); + + CND_CONDITION(!fileExists(name), + "File should not exist immediately after archival."); + return true; + } + + return false; +} + +void TransactionalRAMDirectory::unarchiveOrigFile(const QString& name) +{ + QString origName = filesToRestoreOnAbort.getKey(name); + if (origName.isEmpty()) { + _CLTHROWA(CL_ERR_RAMTransaction, + "File submitted for unarchival was not archived."); + } + RAMFile* origFile = filesToRestoreOnAbort.get(name); + // Transfer memory ownership back to files from filesToRestoreOnAbort. + filesToRestoreOnAbort.remove(name, true, true); + files.put(origName, origFile); +} + +bool TransactionalRAMDirectory::transIsOpen() const +{ + return transOpen; +} + +void TransactionalRAMDirectory::transStart() +{ + if (transOpen) { + _CLTHROWA(CL_ERR_RAMTransaction, + "Must resolve previous transaction before starting another."); + } + + CND_CONDITION(filesToRemoveOnAbort.size() == 0, + "filesToRemoveOnAbort should have been cleared by either its" + " constructor or transResolved."); + + CND_CONDITION(filesToRestoreOnAbort.size() == 0, + "filesToRestoreOnAbort should have been cleared by either its" + " constructor or transResolved."); + + transOpen = true; +} + +void TransactionalRAMDirectory::transResolved() +{ + // This method implements actions common to both forms of transaction + // resolution. + filesToRemoveOnAbort.clear(); + filesToRestoreOnAbort.clear(); + transOpen = false; +} + +void TransactionalRAMDirectory::transCommit() +{ + if (!transOpen) + _CLTHROWA(CL_ERR_RAMTransaction, "There is no open transaction."); + + // All storage is in memory, so commit is ultra-simple. + transResolved(); +} + +void TransactionalRAMDirectory::transAbort() +{ + if (!transOpen) + _CLTHROWA(CL_ERR_RAMTransaction, "There is no open transaction."); + + // Delete each file in filesToRemoveOnAbort. + FilenameSet::const_iterator itrDel = filesToRemoveOnAbort.begin(); + for ( ; itrDel != filesToRemoveOnAbort.end(); ++itrDel) { + size_t nameLength = itrDel->first.length(); + + // Special exception: Refrain from deleting a lock's flag file, as that + // would interfere with the operation of the lock. + if (!(nameLength >= 5 + && itrDel->first.rightRef(5) == QLatin1String(".lock"))) { + RAMDirectory::deleteFile(itrDel->first); + } + } + // Ownership of the memory of both the key and the value never left files, + // so there's no need for a special directive to filesToRemoveOnAbort. + filesToRemoveOnAbort.clear(); + + // Now that any new-since-trans-start files with the same names as + // already-present-at-trans-start files are out of the way, restore each + // file in filesToRestoreOnAbort. + TransFileMap::const_iterator itr = filesToRestoreOnAbort.begin(); + for ( ; itr != filesToRestoreOnAbort.end(); ++itr) { + files.put(itr->first, itr->second); + filesToRestoreOnAbort.remove(itr->first); + } + + CND_CONDITION(filesToRestoreOnAbort.size() == 0, + "filesToRestoreOnAbort should be empty."); + + transResolved(); +} + +bool TransactionalRAMDirectory::doDeleteFile(const QString& name) +{ + if (!transOpen) + return RAMDirectory::doDeleteFile(name); + + bool wasOriginalAndWasArchived = archiveOrigFileIfNecessary(name); + if (!wasOriginalAndWasArchived) { + // The file to be deleted wasn't present at transaction start, so instead + // of archiving it, we delete it the conventional way, making sure to + // erase its record in filesToRemoveOnAbort if it was listed there. + filesToRemoveOnAbort.remove(name); + return RAMDirectory::doDeleteFile(name); + } + return true; +} + +void TransactionalRAMDirectory::renameFile(const QString& from, const QString& to) +{ + // During the review on 2005.03.18, decided not to implement transactional + // renameFile for two reasons: + // a) It's not needed in the limited scenario for which + // TransactionalRAMDirectory was designed (IndexWriter::addDocument and + // subcode). + // b) Supporting renaming during a transaction would add considerable + // bookkeeping overhead, reducing the performance of the overwhelmingly + // typical case (commit) in order to support the rare case (abort). + // + // This was not a thinly disguised punt due to the complication of + // implementing renameFile transactionally; rather, several implementations + // were considered, but it seemed wrongheaded to degrade the performance of + // the typical case based on the mere potential need to support renameFile + // at some future point for the benefit of the atypical case. + if (transOpen) { + _CLTHROWA(CL_ERR_RAMTransaction, + "TransactionalRAMDirectory disallows renameFile during a transaction."); + } + RAMDirectory::renameFile(from, to); +} + +IndexOutput* TransactionalRAMDirectory::createOutput(const QString& name) +{ + if (!transOpen) + return RAMDirectory::createOutput(name); + + bool wasOriginalAndWasArchived = archiveOrigFileIfNecessary(name); + try { + IndexOutput* ret = RAMDirectory::createOutput(name); + // Importantly, we store a pointer to the filename memory managed by + // files, rather than that passed in by the client (name). We don't make + // an additional copy of the filename's memory because the transactional + // metadata container filesToRemoveOnAbort is not at risk of outliving + // files. + filesToRemoveOnAbort.put(files.getKey(name), NULL); + return ret; + } catch (...) { + if (wasOriginalAndWasArchived) { + unarchiveOrigFile(name); + } + throw; + } +} + +void TransactionalRAMDirectory::close() +{ + if (transOpen) + transAbort(); + + RAMDirectory::close(); +} + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h b/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h new file mode 100644 index 000000000..44c5e8e99 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_TransactionalRAMDirectory_ +#define _lucene_store_TransactionalRAMDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "RAMDirectory.h" +#include "CLucene/util/VoidList.h" + +CL_NS_DEF(store) + +/*** +This transactional in-memory Directory was created to address a specific +situation, and was deliberately pared down to the simplest viable +implementation. For the sake of simplicity, this implementation imposes +restrictions on what operations can be performed in the directory while a +transaction is in progress (documented in TransactionalRAMDirectory.cpp). + +Because the Lucene Directory interface itself is rather simplistic, it +would not be difficult to expand TransactionalRAMDirectory so that it +provided fully general transactionality. However, the developer of this +original implementation was of the opinion that the last thing CLucene +needs is gratuitous features that exceed their required complexity and +haven't been rigorously tested. +*/ +class TransactionalRAMDirectory : public RAMDirectory +{ +private: + typedef CL_NS(util)::CLSet<QString, void*, CL_NS(util)::Compare::Qstring, + CL_NS(util)::Deletor::DummyQString> FilenameSet; + FilenameSet filesToRemoveOnAbort; + + typedef CL_NS(util)::CLSet<QString, RAMFile*, CL_NS(util)::Compare::Qstring, + CL_NS(util)::Deletor::DummyQString, + CL_NS(util)::Deletor::Object<RAMFile> > TransFileMap; + TransFileMap filesToRestoreOnAbort; + + bool transOpen; + + void transResolved(); + bool archiveOrigFileIfNecessary(const QString& name); + void unarchiveOrigFile(const QString& name); + +protected: + bool doDeleteFile(const QString& name); + +public: + TransactionalRAMDirectory(); + virtual ~TransactionalRAMDirectory(); + + bool transIsOpen() const; + void transStart(); + void transCommit(); + void transAbort(); + + // Constrained operations: + void renameFile(const QString& from, const QString& to); + IndexOutput* createOutput(const QString& name); + + void close(); +}; + +CL_NS_END + +#endif |