diff options
Diffstat (limited to '3rdparty/clucene/src/CLucene/store')
16 files changed, 3033 insertions, 0 deletions
diff --git a/3rdparty/clucene/src/CLucene/store/Directory.h b/3rdparty/clucene/src/CLucene/store/Directory.h new file mode 100644 index 000000000..818bc7af9 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/Directory.h @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_Directory +#define _lucene_store_Directory + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QStringList> + +#include "CLucene/store/Lock.h" +#include "CLucene/util/VoidList.h" +#include "CLucene/util/Misc.h" + +#include "IndexInput.h" +#include "IndexOutput.h" + +CL_NS_DEF(store) + +/** A Directory is a flat list of files. Files may be written once, when they +* are created. Once a file is created it may only be opened for read, or +* deleted. Random access is permitted both when reading and writing. +* +* <p> Direct i/o is not used directly, but rather all i/o is +* through this API. This permits things such as: <ul> +* <li> implementation of RAM-based indices; +* <li> implementation indices stored in a database, via a database; +* <li> implementation of an index as a single file; +* </ul> +* +*/ +class Directory : LUCENE_REFBASE +{ +protected: + Directory() {} + // Removes an existing file in the directory. + virtual bool doDeleteFile(const QString& name) = 0; + +public: + DEFINE_MUTEX(THIS_LOCK) + + virtual ~Directory() {}; + + // Returns an array of strings, one for each file in the directory. + virtual QStringList list() const = 0; + + // Returns true iff a file with the given name exists. + virtual bool fileExists(const QString& name) const = 0; + + // Returns the time the named file was last modified. + virtual int64_t fileModified(const QString& name) const = 0; + + // Returns the length of a file in the directory. + virtual int64_t fileLength(const QString& name) const = 0; + + // Returns a stream reading an existing file. + virtual IndexInput* openInput(const QString& name) = 0; + virtual IndexInput* openInput(const QString& name, int32_t bufferSize) + { + // didnt overload bufferSize + return openInput(name); + } + + // Set the modified time of an existing file to now. + virtual void touchFile(const QString& name) = 0; + + // Removes an existing file in the directory. + virtual bool deleteFile(const QString& name, const bool throwError = true) { + bool ret = doDeleteFile(name); + if (!ret && throwError) { + char buffer[200]; + _snprintf(buffer, 200, "couldn't delete file %s", + name.toLocal8Bit().constData()); + _CLTHROWA(CL_ERR_IO, buffer); + } + return ret; + } + + // Renames an existing file in the directory. + // If a file already exists with the new name, then it is replaced. + virtual void renameFile(const QString& from, const QString& to) = 0; + + // Creates a new, empty file in the directory with the given name. + // Returns a stream writing this file. + virtual IndexOutput* createOutput(const QString& name) = 0; + + // Construct a {@link Lock}. + // @param name the name of the lock file + virtual LuceneLock* makeLock(const QString& name) = 0; + + // Closes the store. + virtual void close() = 0; + + virtual QString toString() const = 0; + + virtual QString getDirectoryType() const = 0; +}; + +CL_NS_END + +#endif diff --git a/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp b/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp new file mode 100644 index 000000000..5f96e91cd --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp @@ -0,0 +1,662 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#include <QtCore/QDir> +#include <QtCore/QDateTime> +#include <QtCore/QFileInfo> +#include <QtCore/QByteArray> +#include <QtCore/QCryptographicHash> + +#include "CLucene/StdHeader.h" +#include "FSDirectory.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +CL_NS_DEF(store) +CL_NS_USE(util) + +bool FSDirectory::disableLocks = false; + +// This cache of directories ensures that there is a unique Directory instance +// per path, so that synchronization on the Directory can be used to synchronize +// access between readers and writers. +static CL_NS(util)::CLHashMap<QString, FSDirectory*, + CL_NS(util)::Compare::Qstring, CL_NS(util)::Equals::Qstring, + CL_NS(util)::Deletor::DummyQString> DIRECTORIES(false, false); + +// # pragma mark -- FSDirectory::FSLock + +FSDirectory::FSLock::FSLock(const QString& _lockDir, const QString& name) + : lockDir(_lockDir) + , lockFile(_lockDir + QDir::separator() + name) +{ +} + +FSDirectory::FSLock::~FSLock() +{ +} + +bool FSDirectory::FSLock::obtain() +{ + if (disableLocks) + return true; + + if (QFile::exists(lockFile)) + return false; + + QDir dir(lockDir); + if (!dir.exists()) { + if (!dir.mkpath(lockDir)) { + // 34: len of "Couldn't create lock directory: " + char* err = _CL_NEWARRAY( + char, 34 + strlen(lockDir.toLocal8Bit().constData()) + 1); + strcpy(err, "Couldn't create lock directory: "); + strcat(err, lockDir.toLocal8Bit().constData()); + _CLTHROWA_DEL(CL_ERR_IO, err); + } + } + + QFile file(lockFile); + return file.open(QIODevice::ReadWrite); +} + +void FSDirectory::FSLock::release() +{ + if (disableLocks) + return; + + QFile file(lockFile); + file.remove(); +} + +bool FSDirectory::FSLock::isLocked() +{ + if (disableLocks) + return false; + return QFile::exists(lockFile); +} + +QString FSDirectory::FSLock::toString() const +{ + QString ret(QLatin1String("Lock@")); + return ret.append(lockFile); +} + +// # pragma mark -- FSDirectory::FSIndexInput + +FSDirectory::FSIndexInput::FSIndexInput(const QString& path, int32_t bufferSize) + : BufferedIndexInput(bufferSize) +{ + CND_PRECONDITION(!path.isEmpty(), "path is NULL"); + + handle = _CLNEW SharedHandle(); + handle->fhandle.setFileName(path); + handle->fhandle.open(QIODevice::ReadOnly); + + if (handle->fhandle.error() != QFile::NoError) { + switch(handle->fhandle.error()) { + case 1: + _CLTHROWA(CL_ERR_IO, "An error occurred when reading from the file"); + break; + case 2: + _CLTHROWA(CL_ERR_IO, "An error occurred when writing to the file."); + break; + case 5: + _CLTHROWA(CL_ERR_IO, "The file could not be opened."); + break; + case 6: + _CLTHROWA(CL_ERR_IO, "The operation was aborted."); + break; + case 7: + _CLTHROWA(CL_ERR_IO, "A timeout occurred."); + break; + case 8: + _CLTHROWA(CL_ERR_IO, "An unspecified error occurred."); + break; + case 9: + _CLTHROWA(CL_ERR_IO, "The file could not be removed."); + break; + case 10: + _CLTHROWA(CL_ERR_IO, "The file could not be renamed."); + break; + case 11: + _CLTHROWA(CL_ERR_IO, "The position in the file could not be changed."); + break; + case 12: + _CLTHROWA(CL_ERR_IO, "The file could not be resized.e"); + break; + case 13: + _CLTHROWA(CL_ERR_IO, "The file could not be accessed."); + break; + case 14: + _CLTHROWA(CL_ERR_IO, "The file could not be copied."); + break; + case 4: + default: + _CLTHROWA(CL_ERR_IO, "A fatal error occurred."); + } + } + + //Store the file length + handle->_length = handle->fhandle.size(); + handle->_fpos = 0; + this->_pos = 0; +} + +FSDirectory::FSIndexInput::FSIndexInput(const FSIndexInput& other) + : BufferedIndexInput(other) +{ + if (other.handle == NULL) + _CLTHROWA(CL_ERR_NullPointer, "other handle is null"); + + SCOPED_LOCK_MUTEX(*other.handle->THIS_LOCK) + + _pos = other.handle->_fpos; + handle = _CL_POINTER(other.handle); +} + +FSDirectory::FSIndexInput::~FSIndexInput() +{ + FSIndexInput::close(); +} + +void FSDirectory::FSIndexInput::close() +{ + BufferedIndexInput::close(); +#ifdef _LUCENE_THREADMUTEX + if (handle != NULL) { + // Here we have a bit of a problem... We need to lock the handle to + // ensure that we can safely delete the handle... But if we delete the + // handle, then the scoped unlock, won't be able to unlock the mutex... + + // take a reference of the lock object... + _LUCENE_THREADMUTEX* mutex = handle->THIS_LOCK; + //lock the mutex + mutex->lock(); + + // determine if we are about to delete the handle... + bool doUnlock = (handle->__cl_refcount > 1); + // decdelete (deletes if refcount is down to 0) + _CLDECDELETE(handle); + + if (doUnlock) + mutex->unlock(); + else + delete mutex; + } +#else + _CLDECDELETE(handle); +#endif +} + +IndexInput* FSDirectory::FSIndexInput::clone() const +{ + return _CLNEW FSDirectory::FSIndexInput(*this); +} + +void FSDirectory::FSIndexInput::seekInternal(const int64_t position) +{ + CND_PRECONDITION(position >= 0 && position < handle->_length, + "Seeking out of range") + _pos = position; +} + +void FSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) +{ + SCOPED_LOCK_MUTEX(*handle->THIS_LOCK) + + CND_PRECONDITION(handle != NULL, "shared file handle has closed"); + CND_PRECONDITION(handle->fhandle.isOpen(), "file is not open"); + + if (handle->_fpos != _pos) { + handle->fhandle.seek(_pos); + if (handle->fhandle.pos() != _pos) + _CLTHROWA( CL_ERR_IO, "File IO Seek error"); + handle->_fpos = _pos; + } + + bufferLength = (int32_t)handle->fhandle.read((char*)b, len); + if (bufferLength == 0) + _CLTHROWA(CL_ERR_IO, "read past EOF"); + + if (bufferLength == -1) + _CLTHROWA(CL_ERR_IO, "read error"); + + _pos += bufferLength; + handle->_fpos =_pos; +} + +// # pragma mark -- FSDirectory::FSIndexInput::SharedHandle + +FSDirectory::FSIndexInput::SharedHandle::SharedHandle() + : _fpos(0) + , _length(0) +{ +#ifdef _LUCENE_THREADMUTEX + THIS_LOCK = new _LUCENE_THREADMUTEX; +#endif +} + +FSDirectory::FSIndexInput::SharedHandle::~SharedHandle() +{ + if (fhandle.isOpen()) + fhandle.close(); +} + +// # pragma mark -- FSDirectory::FSIndexOutput + +FSDirectory::FSIndexOutput::FSIndexOutput(const QString& path) +{ + //O_BINARY - Opens file in binary (untranslated) mode + //O_CREAT - Creates and opens new file for writing. Has no effect if file specified by filename exists + //O_RANDOM - Specifies that caching is optimized for, but not restricted to, random access from disk. + //O_WRONLY - Opens file for writing only; + fhandle.setFileName(path); + fhandle.open(QIODevice::ReadWrite | QIODevice::Truncate); + + if (fhandle.error() != QFile::NoError) { + switch(fhandle.error()) { + case 1: + _CLTHROWA(CL_ERR_IO, "An error occurred when reading from the file"); + break; + case 2: + _CLTHROWA(CL_ERR_IO, "An error occurred when writing to the file."); + break; + case 5: + _CLTHROWA(CL_ERR_IO, "The file could not be opened."); + break; + case 6: + _CLTHROWA(CL_ERR_IO, "The operation was aborted."); + break; + case 7: + _CLTHROWA(CL_ERR_IO, "A timeout occurred."); + break; + case 8: + _CLTHROWA(CL_ERR_IO, "An unspecified error occurred."); + break; + case 9: + _CLTHROWA(CL_ERR_IO, "The file could not be removed."); + break; + case 10: + _CLTHROWA(CL_ERR_IO, "The file could not be renamed."); + break; + case 11: + _CLTHROWA(CL_ERR_IO, "The position in the file could not be changed."); + break; + case 12: + _CLTHROWA(CL_ERR_IO, "The file could not be resized.e"); + break; + case 13: + _CLTHROWA(CL_ERR_IO, "The file could not be accessed."); + break; + case 14: + _CLTHROWA(CL_ERR_IO, "The file could not be copied."); + break; + case 4: + default: + _CLTHROWA(CL_ERR_IO, "A fatal error occurred."); + } + } +} + +FSDirectory::FSIndexOutput::~FSIndexOutput() +{ + if (fhandle.isOpen()) { + try { + FSIndexOutput::close(); + } catch (CLuceneError& err) { + //ignore IO errors... + if (err.number() != CL_ERR_IO) + throw; + } + } +} + +void FSDirectory::FSIndexOutput::close() +{ + try { + BufferedIndexOutput::close(); + } catch (CLuceneError& err) { + //ignore IO errors... + if (err.number() != CL_ERR_IO) + throw; + } + fhandle.close(); +} + +int64_t FSDirectory::FSIndexOutput::length() +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + return fhandle.size(); +} + +void FSDirectory::FSIndexOutput::seek(const int64_t pos) +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + + BufferedIndexOutput::seek(pos); + fhandle.seek(pos); + if (fhandle.pos() != pos) + _CLTHROWA(CL_ERR_IO, "File IO Seek error"); +} + +void FSDirectory::FSIndexOutput::flushBuffer(const uint8_t* b, const int32_t size) +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + + if (size > 0 && fhandle.write((const char*)b, size) != size) + _CLTHROWA(CL_ERR_IO, "File IO Write error"); +} + +// # pragma mark -- FSDirectory + +FSDirectory::FSDirectory(const QString& path, const bool createDir) + : Directory() + , refCount(0) + , useMMap(false) +{ + //set a realpath so that if we change directory, we can still function + directory = QFileInfo(path).absoluteFilePath(); + lockDir = directory; + + QDir dir(lockDir); + if (!dir.exists()) { + if (!dir.mkpath(lockDir)) + _CLTHROWA_DEL(CL_ERR_IO, "Cannot create temp directory"); + } + + QFileInfo info(lockDir); + if (info.isFile() || info.isSymLink()) + _CLTHROWA(CL_ERR_IO, "Found regular file where directory expected"); + + if (createDir) + create(); + + dir.setPath(directory); + if (!dir.exists()) { + //19: len of " is not a directory" + char* err = + _CL_NEWARRAY(char, 19 + strlen(path.toLocal8Bit().constData()) + 1); + strcpy(err, path.toLocal8Bit().constData()); + strcat(err, " is not a directory"); + _CLTHROWA_DEL(CL_ERR_IO, err); + } +} + +void FSDirectory::create() +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + + bool clear = false; + QDir dir(directory); + if (!dir.exists()) { + if (!dir.mkpath(directory)) { + char* err = _CL_NEWARRAY( // 27 len of "Couldn't create directory:" + char, 27 + strlen(directory.toLocal8Bit().constData()) + 1); + strcpy(err, "Couldn't create directory: "); + strcat(err, directory.toLocal8Bit().constData()); + _CLTHROWA_DEL(CL_ERR_IO, err); + } + } else { + clear = true; + } + + QFileInfo info(directory); + if (info.isFile() || info.isSymLink()) { + char tmp[1024]; + _snprintf(tmp, 1024, "%s not a directory", + directory.toLocal8Bit().constData()); + _CLTHROWA(CL_ERR_IO, tmp); + } + + if (clear) { + dir.setPath(directory); + // clear probably existing lucene index files + QStringList fileList = dir.entryList(QDir::Files | QDir::Hidden + | QDir::NoSymLinks); + foreach(const QString file, fileList) { + if (CL_NS(index)::IndexReader::isLuceneFile(file)) { + if (!dir.remove(file)) + _CLTHROWA(CL_ERR_IO, "Couldn't delete file "); + } + } + + // clear probably existing file locks + QFileInfo dirInfo(lockDir); + if (dirInfo.exists() && dirInfo.isReadable() && dirInfo.isWritable() + && !dirInfo.isFile() && !dirInfo.isSymLink()) { + QDir lockDirectory(lockDir); + fileList = dir.entryList(QStringList() << getLockPrefix() + + QLatin1Char('*'), QDir::Files | QDir::Hidden | QDir::NoSymLinks); + + foreach(const QString file, fileList) { + if (!lockDirectory.remove(file)) + _CLTHROWA(CL_ERR_IO, "Couldn't delete file "); + } + } + else { + //todo: richer error: + lockDir.getAbsolutePath()); + _CLTHROWA(CL_ERR_IO, "Cannot read lock directory"); + } + } +} + +void FSDirectory::priv_getFN(QString& buffer, const QString& name) const +{ + buffer.clear(); + buffer.append(directory); + buffer.append(QDir::separator()); + buffer.append(name); +} + +FSDirectory::~FSDirectory() +{ +} + +QStringList FSDirectory::list() const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.entryList(QDir::Files | QDir::Hidden); +} + +bool FSDirectory::fileExists(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.entryList().contains(name); +} + +QString FSDirectory::getDirName() const +{ + return directory; +} + +//static +FSDirectory* FSDirectory::getDirectory(const QString& file, const bool _create) +{ + FSDirectory* dir = NULL; + { + if (file.isEmpty()) + _CLTHROWA(CL_ERR_IO, "Invalid directory"); + + SCOPED_LOCK_MUTEX(DIRECTORIES.THIS_LOCK) + dir = DIRECTORIES.get(file); + if ( dir == NULL ){ + dir = _CLNEW FSDirectory(file, _create); + DIRECTORIES.put(dir->directory, dir); + } else if (_create) { + dir->create(); + } + + { + SCOPED_LOCK_MUTEX(dir->THIS_LOCK) + dir->refCount++; + } + } + + return _CL_POINTER(dir); +} + +int64_t FSDirectory::fileModified(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFileInfo fInfo(directory + QDir::separator() + name); + return fInfo.lastModified().toTime_t(); +} + +//static +int64_t FSDirectory::fileModified(const QString& dir, const QString& name) +{ + QFileInfo fInfo(dir + QDir::separator() + name); + return fInfo.lastModified().toTime_t(); +} + +void FSDirectory::touchFile(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFile file(directory + QDir::separator() + name); + if (!file.open(QIODevice::ReadWrite)) + _CLTHROWA(CL_ERR_IO, "IO Error while touching file"); +} + +int64_t FSDirectory::fileLength(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFileInfo fInfo(directory + QDir::separator() + name); + return fInfo.size(); +} + +IndexInput* FSDirectory::openInput(const QString& name) +{ + return openInput(name, CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); +} + +IndexInput* FSDirectory::openInput(const QString& name, int32_t bufferSize ) +{ + CND_PRECONDITION(directory[0]!=0,"directory is not open") + + return _CLNEW FSIndexInput(directory + QDir::separator() + name, bufferSize); +} + +void FSDirectory::close() +{ + SCOPED_LOCK_MUTEX(DIRECTORIES.THIS_LOCK) + { + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + //refcount starts at 1 + if (--refCount <= 0) { + Directory* dir = DIRECTORIES.get(getDirName()); + if (dir) { + //this will be removed in ~FSDirectory + DIRECTORIES.remove(getDirName()); + _CLDECDELETE(dir); + } + } + } +} + +QString FSDirectory::getLockPrefix() const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QString dirName(QFileInfo(directory).absoluteFilePath()); + if (dirName.isEmpty()) + _CLTHROWA(CL_ERR_Runtime, "Invalid directory path"); + + // to be compatible with jlucene, + // we need to make some changes ... + if (dirName.at(1) == QLatin1Char(':')) + dirName[0] = dirName.at(0).toUpper(); + + TCHAR tBuffer[2048] = { 0 }; + dirName.toWCharArray(tBuffer); + + char aBuffer[4096] = { 0 }; + STRCPY_TtoA(aBuffer, tBuffer, 4096); + + QString string(QLatin1String("lucene-")); + QByteArray hash(QCryptographicHash::hash(aBuffer, QCryptographicHash::Md5)); + + // TODO: verify this !!! + return string.append(QLatin1String(hash.toHex().constData())); +} + +bool FSDirectory::doDeleteFile(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.remove(name); +} + +void FSDirectory::renameFile(const QString& from, const QString& to) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + SCOPED_LOCK_MUTEX(THIS_LOCK) + + if (fileExists(to)) + deleteFile(to, false); + + QFile file(directory + QDir::separator() + from); + QString newFile(directory + QDir::separator() + to); + if (!file.rename(newFile)) { + // try a second time if we fail + if (fileExists(to)) + deleteFile(to, false); + + if (!file.rename(newFile)) { + QString error(QLatin1String("Could not rename: %1 to %2!!!!")); + error.arg(from).arg(newFile); + QByteArray bArray(error.toLocal8Bit()); + _CLTHROWA(CL_ERR_IO, bArray.constData()); + } + } +} + +IndexOutput* FSDirectory::createOutput(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QString file = directory + QDir::separator() + name; + if (QFileInfo(file).exists()) { + if (!QFile::remove(file)) { + QByteArray bArray("Cannot overwrite: "); + bArray.append(name.toLocal8Bit()); + _CLTHROWA(CL_ERR_IO, bArray.constData()); + } + } + return _CLNEW FSIndexOutput(file); +} + +LuceneLock* FSDirectory::makeLock(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + + QString lockFile(getLockPrefix()); + lockFile.append(QLatin1Char('-')).append(name); + + return _CLNEW FSLock(lockDir, lockFile); +} + +QString FSDirectory::toString() const +{ + return QString::fromLatin1("FSDirectory@").append(directory); +} + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/FSDirectory.h b/3rdparty/clucene/src/CLucene/store/FSDirectory.h new file mode 100644 index 000000000..e967380e0 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/FSDirectory.h @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_FSDirectory_ +#define _lucene_store_FSDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QFile> +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "Directory.h" +#include "Lock.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_DEF(store) + +/** +* Straightforward implementation of {@link Directory} as a directory of files. +* <p>If the system property 'disableLuceneLocks' has the String value of +* "true", lock creation will be disabled. +* +* @see Directory +*/ +class FSDirectory : public Directory +{ +public: + // Destructor - only call this if you are sure the directory + // is not being used anymore. Otherwise use the ref-counting + // facilities of _CLDECDELETE + ~FSDirectory(); + + // Get a list of strings, one for each file in the directory. + QStringList list() const; + + // Returns true iff a file with the given name exists. + bool fileExists(const QString& name) const; + + // Returns the text name of the directory + QString getDirName() const; ///<returns reference + + /** + Returns the directory instance for the named location. + + Do not delete this instance, only use close, otherwise other instances + will lose this instance. + + <p>Directories are cached, so that, for a given canonical path, the same + FSDirectory instance will always be returned. This permits + synchronization on directories. + + @param file the path to the directory. + @param create if true, create, or erase any existing contents. + @return the FSDirectory for the named file. + */ + static FSDirectory* getDirectory(const QString& file, const bool create); + + // Returns the time the named file was last modified. + int64_t fileModified(const QString& name) const; + + //static + // Returns the time the named file was last modified. + static int64_t fileModified(const QString& dir, const QString& name); + + // static + // Returns the length in bytes of a file in the directory. + int64_t fileLength(const QString& name) const; + + // Returns a stream reading an existing file. + IndexInput* openInput(const QString& name); + IndexInput* openInput(const QString& name, int32_t bufferSize); + + // Renames an existing file in the directory. + void renameFile(const QString& from, const QString& to); + + // Set the modified time of an existing file to now. + void touchFile(const QString& name); + + // Creates a new, empty file in the directory with the given name. + // Returns a stream writing this file. + IndexOutput* createOutput(const QString& name); + + // Construct a {@link Lock}. + // @param name the name of the lock file + LuceneLock* makeLock(const QString& name); + + // Decrease the ref-count to the directory by one. If the object is no + // longer needed, then the object is removed from the directory pool. + void close(); + + // If MMap is available, this can disable use of mmap reading. + void setUseMMap(bool value) { useMMap = value; } + + // Gets whether the directory is using MMap for inputstreams. + bool getUseMMap() const { return useMMap; } + + QString toString() const; + + static QString DirectoryType() { return QLatin1String("FS"); } + QString getDirectoryType() const { return QLatin1String("FS"); } + + // Set whether Lucene's use of lock files is disabled. By default, + // lock files are enabled. They should only be disabled if the index + // is on a read-only medium like a CD-ROM. + static void setDisableLocks(bool doDisableLocks) + { disableLocks = doDisableLocks; } + + // Returns whether Lucene's use of lock files is disabled. + // @return true if locks are disabled, false if locks are enabled. + static bool getDisableLocks() { return disableLocks; } + +protected: + FSDirectory(const QString& path, const bool createDir); + // Removes an existing file in the directory. + bool doDeleteFile(const QString& name); + +private: + class FSLock : public LuceneLock { + public: + FSLock (const QString& lockDir, const QString& name); + ~FSLock(); + + bool obtain(); + void release(); + bool isLocked(); + QString toString() const; + + QString lockDir; + QString lockFile; + }; + friend class FSDirectory::FSLock; + + class FSIndexInput : public BufferedIndexInput { + public: + FSIndexInput(const QString& path, int32_t bufferSize = + CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + ~FSIndexInput(); + + void close(); + IndexInput* clone() const; + + int64_t length() + { return handle->_length; } + + QString getDirectoryType() const + { return FSDirectory::DirectoryType(); } + + protected: + FSIndexInput(const FSIndexInput& clone); + // Random-access methods + void seekInternal(const int64_t position); + // IndexInput methods + void readInternal(uint8_t* b, const int32_t len); + + private: + // We used a shared handle between all the fsindexinput clones. + // This reduces number of file handles we need, and it means + // we dont have to use file tell (which is slow) before doing a read. + class SharedHandle : LUCENE_REFBASE { + public: + SharedHandle(); + ~SharedHandle(); + + int64_t _fpos; + int64_t _length; + + QFile fhandle; + DEFINE_MUTEX(*THIS_LOCK) + }; + SharedHandle* handle; + int64_t _pos; + }; + friend class FSDirectory::FSIndexInput; + + class FSIndexOutput : public BufferedIndexOutput { + public: + FSIndexOutput(const QString& path); + ~FSIndexOutput(); + + void close(); + int64_t length(); + void seek(const int64_t pos); + + protected: + void flushBuffer(const uint8_t* b, const int32_t size); + + private: + QFile fhandle; + }; + friend class FSDirectory::FSIndexOutput; + +private: + QString directory; + int refCount; + void create(); + + QString lockDir; + QString getLockPrefix() const; + static bool disableLocks; + + void priv_getFN(QString& buffer, const QString& name) const; + bool useMMap; +}; + +CL_NS_END + +#endif diff --git a/3rdparty/clucene/src/CLucene/store/IndexInput.cpp b/3rdparty/clucene/src/CLucene/store/IndexInput.cpp new file mode 100644 index 000000000..cf7bd16b7 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/IndexInput.cpp @@ -0,0 +1,233 @@ + /*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "IndexInput.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + + IndexInput::IndexInput() + { + } + IndexInput::IndexInput(const IndexInput& other) + { + } + + int32_t IndexInput::readInt() { + int32_t b = (readByte() << 24); + b |= (readByte() << 16); + b |= (readByte() << 8); + return (b | readByte()); + } + + int32_t IndexInput::readVInt() { + uint8_t b = readByte(); + int32_t i = b & 0x7F; + for (int32_t shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (b & 0x7F) << shift; + } + return i; + } + + int64_t IndexInput::readLong() { + int64_t i = ((int64_t)readInt() << 32); + return (i | ((int64_t)readInt() & 0xFFFFFFFFL)); + } + + int64_t IndexInput::readVLong() { + uint8_t b = readByte(); + int64_t i = b & 0x7F; + for (int32_t shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (((int64_t)b) & 0x7FL) << shift; + } + return i; + } + + void IndexInput::skipChars( const int32_t count) { + for (int32_t i = 0; i < count; i++) { + TCHAR b = readByte(); + if ((b & 0x80) == 0) { + // Do Nothing. + } else if ((b & 0xE0) != 0xE0) { + readByte(); + } else { + readByte(); + readByte(); + } + } + } + + int32_t IndexInput::readString(TCHAR* buffer, const int32_t maxLength){ + int32_t len = readVInt(); + int32_t ml=maxLength-1; + if ( len >= ml ){ + readChars(buffer, 0, ml); + buffer[ml] = 0; + //we have to finish reading all the data for this string! + if ( len-ml > 0 ){ + //seek(getFilePointer()+(len-ml)); <- that was the wrong way to "finish reading" + skipChars(len-ml); + } + return ml; + }else{ + readChars(buffer, 0, len); + buffer[len] = 0; + return len; + } + } + + TCHAR* IndexInput::readString(const bool _unique){ + int32_t len = readVInt(); + + if ( len == 0){ + if ( _unique ) //todo: does non unique ever occur? + return stringDuplicate(LUCENE_BLANK_STRING); + else + return LUCENE_BLANK_STRING; + } + + TCHAR* ret = _CL_NEWARRAY(TCHAR,len+1); + readChars(ret, 0, len); + ret[len] = 0; + + return ret; + } + + void IndexInput::readChars( TCHAR* buffer, const int32_t start, const int32_t len) { + const int32_t end = start + len; + TCHAR b; + for (int32_t i = start; i < end; ++i) { + b = readByte(); + if ((b & 0x80) == 0) { + b = (b & 0x7F); + } else if ((b & 0xE0) != 0xE0) { + b = (((b & 0x1F) << 6) + | (readByte() & 0x3F)); + } else { + b = ((b & 0x0F) << 12) | ((readByte() & 0x3F) << 6); + b |= (readByte() & 0x3F); + } + buffer[i] = b; + } + } + + + + + + +BufferedIndexInput::BufferedIndexInput(int32_t _bufferSize): + buffer(NULL), + bufferSize(_bufferSize), + bufferStart(0), + bufferLength(0), + bufferPosition(0) + { + } + + BufferedIndexInput::BufferedIndexInput(const BufferedIndexInput& other): + IndexInput(other), + buffer(NULL), + bufferSize(other.bufferSize), + bufferStart(other.bufferStart), + bufferLength(other.bufferLength), + bufferPosition(other.bufferPosition) + { + /* DSR: Does the fact that sometime clone.buffer is not NULL even when + ** clone.bufferLength is zero indicate memory corruption/leakage? + ** if ( clone.buffer != NULL) { */ + if (other.bufferLength != 0 && other.buffer != NULL) { + buffer = _CL_NEWARRAY(uint8_t,bufferLength); + memcpy(buffer,other.buffer,bufferLength * sizeof(uint8_t)); + } + } + + void BufferedIndexInput::readBytes(uint8_t* b, const int32_t len){ + if (len < bufferSize) { + for (int32_t i = 0; i < len; ++i) // read byte-by-byte + b[i] = readByte(); + } else { // read all-at-once + int64_t start = getFilePointer(); + seekInternal(start); + readInternal(b, len); + + bufferStart = start + len; // adjust stream variables + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read + } + } + + int64_t BufferedIndexInput::getFilePointer() const{ + return bufferStart + bufferPosition; + } + + void BufferedIndexInput::seek(const int64_t pos) { + if ( pos < 0 ) + _CLTHROWA(CL_ERR_IO, "IO Argument Error. Value must be a positive value."); + if (pos >= bufferStart && pos < (bufferStart + bufferLength)) + bufferPosition = (int32_t)(pos - bufferStart); // seek within buffer + else { + bufferStart = pos; + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read() + seekInternal(pos); + } + } + void BufferedIndexInput::close(){ + _CLDELETE_ARRAY(buffer); + bufferLength = 0; + bufferPosition = 0; + bufferStart = 0; + } + + + BufferedIndexInput::~BufferedIndexInput(){ + BufferedIndexInput::close(); + } + + void BufferedIndexInput::refill() { + int64_t start = bufferStart + bufferPosition; + int64_t end = start + bufferSize; + if (end > length()) // don't read past EOF + end = length(); + bufferLength = (int32_t)(end - start); + if (bufferLength == 0) + _CLTHROWA(CL_ERR_IO, "IndexInput read past EOF"); + + if (buffer == NULL){ + buffer = _CL_NEWARRAY(uint8_t,bufferSize); // allocate buffer lazily + } + readInternal(buffer, bufferLength); + + + bufferStart = start; + bufferPosition = 0; + } + + +IndexInputStream::IndexInputStream(IndexInput* input){ + this->input = input; + this->size = input->length(); + this->position = input->getFilePointer(); +} +IndexInputStream::~IndexInputStream(){ +} +int32_t IndexInputStream::fillBuffer(char* start, int32_t space){ + int64_t avail = input->length()-input->getFilePointer(); + if ( avail == 0 ) + return -1; + else if ( avail<space ) + space = (int32_t)avail; + + input->readBytes((uint8_t*)start,space); + return space; +} + +CL_NS_END + diff --git a/3rdparty/clucene/src/CLucene/store/IndexInput.h b/3rdparty/clucene/src/CLucene/store/IndexInput.h new file mode 100644 index 000000000..9453b5cf1 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/IndexInput.h @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_IndexInput_ +#define _lucene_store_IndexInput_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/util/bufferedstream.h" +#include "IndexOutput.h" + +CL_NS_DEF(store) + + /** Abstract base class for input from a file in a {@link Directory}. A + * random-access input stream. Used for all Lucene index input operations. + * @see Directory + * @see IndexOutput + */ + class IndexInput: LUCENE_BASE { + private: + void skipChars( const int32_t count); + protected: + IndexInput(); + IndexInput(const IndexInput& clone); + public: + virtual ~IndexInput(){} + virtual IndexInput* clone() const =0; + + DEFINE_MUTEX(THIS_LOCK) + + /** Reads and returns a single byte. + * @see IndexOutput#writeByte(byte) + */ + virtual uint8_t readByte() =0; + + /** Reads a specified number of bytes into an array at the specified offset. + * @param b the array to read bytes into + * @param offset the offset in the array to start storing bytes + * @param len the number of bytes to read + * @see IndexOutput#writeBytes(byte[],int32_t) + */ + virtual void readBytes(uint8_t* b, const int32_t len) =0; + + /** Reads four bytes and returns an int. + * @see IndexOutput#writeInt(int32_t) + */ + int32_t readInt(); + + /** Reads an int stored in variable-length format. Reads between one and + * five bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexOutput#writeVInt(int32_t) + */ + virtual int32_t readVInt(); + + /** Reads eight bytes and returns a long. + * @see IndexOutput#writeLong(long) + */ + int64_t readLong(); + + /** Reads a long stored in variable-length format. Reads between one and + * nine bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. */ + int64_t readVLong(); + + /** Reads a string. + * @see IndexOutput#writeString(String) + * maxLength is the amount read into the buffer, the whole string is still read from the stream + * returns the amount read + */ + int32_t readString(TCHAR* buffer, const int32_t maxlength); + + /** Reads a string. + * @see IndexOutput#writeString(String) + * If unique is true (default) the string will be duplicated. + * If false and the length is zero, LUCENE_BLANK_STRING is returned + */ + TCHAR* readString(const bool unique=true); + + + /** Reads UTF-8 encoded characters into an array. + * @param buffer the array to read characters into + * @param start the offset in the array to start storing characters + * @param length the number of characters to read + * @see IndexOutput#writeChars(String,int32_t,int32_t) + */ + void readChars( TCHAR* buffer, const int32_t start, const int32_t len); + + /** Closes the stream to futher operations. */ + virtual void close() =0; + + /** Returns the current position in this file, where the next read will + * occur. + * @see #seek(long) + */ + virtual int64_t getFilePointer() const =0; + + /** Sets current position in this file, where the next read will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos) =0; + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + virtual QString getDirectoryType() const = 0; + }; + + /** Abstract base class for input from a file in a {@link Directory}. A + * random-access input stream. Used for all Lucene index input operations. + * @see Directory + * @see IndexOutput + */ + class BufferedIndexInput: public IndexInput{ + private: + uint8_t* buffer; //array of bytes + void refill(); + protected: + int32_t bufferSize; //size of the buffer + int64_t bufferStart; // position in file of buffer + int32_t bufferLength; // end of valid l_byte_ts + int32_t bufferPosition; // next uint8_t to read + + /** Returns a clone of this stream. + * + * <p>Clones of a stream access the same data, and are positioned at the same + * point as the stream they were cloned from. + * + * <p>Expert: Subclasses must ensure that clones may be positioned at + * different points in the input from each other and from the stream they + * were cloned from. + */ + BufferedIndexInput(const BufferedIndexInput& clone); + BufferedIndexInput(int32_t bufferSize = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + public: + + virtual ~BufferedIndexInput(); + virtual IndexInput* clone() const = 0; + void close(); + inline uint8_t readByte(){ + if (bufferPosition >= bufferLength) + refill(); + + return buffer[bufferPosition++]; + } + void readBytes(uint8_t* b, const int32_t len); + int64_t getFilePointer() const; + void seek(const int64_t pos); + + protected: + /** Expert: implements buffer refill. Reads bytes from the current position + * in the input. + * @param b the array to read bytes into + * @param offset the offset in the array to start storing bytes + * @param length the number of bytes to read + */ + virtual void readInternal(uint8_t* b, const int32_t len) = 0; + + /** Expert: implements seek. Sets current position in this file, where the + * next {@link #readInternal(byte[],int32_t,int32_t)} will occur. + * @see #readInternal(byte[],int32_t,int32_t) + */ + virtual void seekInternal(const int64_t pos) = 0; + }; + + /** + * JStream InputStream which reads from an IndexInput. This class is + * used by the FieldReader to create binary fields. You can then use + * a GZipInputStream to read compressed data or any of the other + * JStream stream types. + * + */ + class IndexInputStream: public jstreams::BufferedInputStream<char>{ + IndexInput* input; + public: + IndexInputStream(IndexInput* input); + ~IndexInputStream(); + int32_t fillBuffer(char* start, int32_t space); + }; +CL_NS_END +#endif diff --git a/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp b/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp new file mode 100644 index 000000000..04f78c348 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp @@ -0,0 +1,163 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "IndexOutput.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + + + IndexOutput::IndexOutput() + { + } + + IndexOutput::~IndexOutput(){ + } + + BufferedIndexOutput::BufferedIndexOutput() + { + buffer = _CL_NEWARRAY(uint8_t, BUFFER_SIZE ); + bufferStart = 0; + bufferPosition = 0; + } + + BufferedIndexOutput::~BufferedIndexOutput(){ + if ( buffer != NULL ) + close(); + } + + void BufferedIndexOutput::close(){ + flush(); + _CLDELETE_ARRAY( buffer ); + + bufferStart = 0; + bufferPosition = 0; + } + + void BufferedIndexOutput::writeByte(const uint8_t b) { + CND_PRECONDITION(buffer!=NULL,"IndexOutput is closed") + if (bufferPosition >= BUFFER_SIZE) + flush(); + buffer[bufferPosition++] = b; + } + + void BufferedIndexOutput::writeBytes(const uint8_t* b, const int32_t length) { + if ( length < 0 ) + _CLTHROWA(CL_ERR_IllegalArgument, "IO Argument Error. Value must be a positive value."); + int32_t bytesLeft = BUFFER_SIZE - bufferPosition; + // is there enough space in the buffer? + if (bytesLeft >= length) { + // we add the data to the end of the buffer + memcpy(buffer + bufferPosition, b, length); + bufferPosition += length; + // if the buffer is full, flush it + if (BUFFER_SIZE - bufferPosition == 0) + flush(); + } else { + // is data larger then buffer? + if (length > BUFFER_SIZE) { + // we flush the buffer + if (bufferPosition > 0) + flush(); + // and write data at once + flushBuffer(b, length); + bufferStart += length; + } else { + // we fill/flush the buffer (until the input is written) + int64_t pos = 0; // position in the input data + int32_t pieceLength; + while (pos < length) { + if ( length - pos < bytesLeft ) + pieceLength = length - pos; + else + pieceLength = bytesLeft; + memcpy(buffer + bufferPosition, b + pos, pieceLength); + pos += pieceLength; + bufferPosition += pieceLength; + // if the buffer is full, flush it + bytesLeft = BUFFER_SIZE - bufferPosition; + if (bytesLeft == 0) { + flush(); + bytesLeft = BUFFER_SIZE; + } + } + } + } + } + + void IndexOutput::writeInt(const int32_t i) { + writeByte((uint8_t)(i >> 24)); + writeByte((uint8_t)(i >> 16)); + writeByte((uint8_t)(i >> 8)); + writeByte((uint8_t) i); + } + + void IndexOutput::writeVInt(const int32_t vi) { + uint32_t i = vi; + while ((i & ~0x7F) != 0) { + writeByte((uint8_t)((i & 0x7f) | 0x80)); + i >>= 7; //doing unsigned shift + } + writeByte( (uint8_t)i ); + } + + void IndexOutput::writeLong(const int64_t i) { + writeInt((int32_t) (i >> 32)); + writeInt((int32_t) i); + } + + void IndexOutput::writeVLong(const int64_t vi) { + uint64_t i = vi; + while ((i & ~0x7F) != 0) { + writeByte((uint8_t)((i & 0x7f) | 0x80)); + i >>= 7; //doing unsigned shift + } + writeByte((uint8_t)i); + } + + void IndexOutput::writeString(const TCHAR* s, const int32_t length ) { + writeVInt(length); + writeChars(s, 0, length); + } + + void IndexOutput::writeChars(const TCHAR* s, const int32_t start, const int32_t length){ + if ( length < 0 || start < 0 ) + _CLTHROWA(CL_ERR_IllegalArgument, "IO Argument Error. Value must be a positive value."); + + const int32_t end = start + length; + for (int32_t i = start; i < end; ++i) { + const int32_t code = (int32_t)s[i]; + if (code >= 0x01 && code <= 0x7F) + writeByte((uint8_t)code); + else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) { + writeByte((uint8_t)(0xC0 | (code >> 6))); + writeByte((uint8_t)(0x80 | (code & 0x3F))); + } else { + writeByte((uint8_t)(0xE0 | (((uint32_t)code) >> 12))); //unsigned shift + writeByte((uint8_t)(0x80 | ((code >> 6) & 0x3F))); + writeByte((uint8_t)(0x80 | (code & 0x3F))); + } + } + } + + + int64_t BufferedIndexOutput::getFilePointer() const{ + return bufferStart + bufferPosition; + } + + void BufferedIndexOutput::seek(const int64_t pos) { + flush(); + bufferStart = pos; + } + + void BufferedIndexOutput::flush() { + flushBuffer(buffer, bufferPosition); + bufferStart += bufferPosition; + bufferPosition = 0; + } + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/IndexOutput.h b/3rdparty/clucene/src/CLucene/store/IndexOutput.h new file mode 100644 index 000000000..c47ee73a7 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/IndexOutput.h @@ -0,0 +1,152 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_IndexOutput_ +#define _lucene_store_IndexOutput_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + + +/** Abstract class for output to a file in a Directory. A random-access output +* stream. Used for all Lucene index output operations. +* @see Directory +* @see IndexInput +*/ +class IndexOutput:LUCENE_BASE{ + bool isclosed; +public: + IndexOutput(); + virtual ~IndexOutput(); + + /** Writes a single byte. + * @see IndexInput#readByte() + */ + virtual void writeByte(const uint8_t b) = 0; + + /** Writes an array of bytes. + * @param b the bytes to write + * @param length the number of bytes to write + * @see IndexInput#readBytes(byte[],int32_t,int32_t) + */ + virtual void writeBytes(const uint8_t* b, const int32_t length) = 0; + + /** Writes an int as four bytes. + * @see IndexInput#readInt() + */ + void writeInt(const int32_t i); + + /** Writes an int in a variable-length format. Writes between one and + * five bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexInput#readVInt() + */ + void writeVInt(const int32_t vi); + + /** Writes a long as eight bytes. + * @see IndexInput#readLong() + */ + void writeLong(const int64_t i); + + /** Writes an long in a variable-length format. Writes between one and five + * bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexInput#readVLong() + */ + void writeVLong(const int64_t vi); + + /** Writes a string. + * @see IndexInput#readString() + */ + void writeString(const TCHAR* s, const int32_t length); + + /** Writes a sequence of UTF-8 encoded characters from a string. + * @param s the source of the characters + * @param start the first character in the sequence + * @param length the number of characters in the sequence + * @see IndexInput#readChars(char[],int32_t,int32_t) + */ + void writeChars(const TCHAR* s, const int32_t start, const int32_t length); + + /** Closes this stream to further operations. */ + virtual void close() = 0; + + /** Returns the current position in this file, where the next write will + * occur. + * @see #seek(long) + */ + virtual int64_t getFilePointer() const = 0; + + /** Sets current position in this file, where the next write will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos) = 0; + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + /** Forces any buffered output to be written. */ + virtual void flush() = 0; +}; + +/** Base implementation class for buffered {@link IndexOutput}. */ +class BufferedIndexOutput : public IndexOutput{ +public: + LUCENE_STATIC_CONSTANT(int32_t, BUFFER_SIZE=LUCENE_STREAM_BUFFER_SIZE); +private: + uint8_t* buffer; + int64_t bufferStart; // position in file of buffer + int32_t bufferPosition; // position in buffer + +public: + BufferedIndexOutput(); + virtual ~BufferedIndexOutput(); + + /** Writes a single byte. + * @see IndexInput#readByte() + */ + virtual void writeByte(const uint8_t b); + + /** Writes an array of bytes. + * @param b the bytes to write + * @param length the number of bytes to write + * @see IndexInput#readBytes(byte[],int32_t,int32_t) + */ + virtual void writeBytes(const uint8_t* b, const int32_t length); + + /** Closes this stream to further operations. */ + virtual void close(); + + /** Returns the current position in this file, where the next write will + * occur. + * @see #seek(long) + */ + int64_t getFilePointer() const; + + /** Sets current position in this file, where the next write will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos); + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + /** Forces any buffered output to be written. */ + void flush(); + +protected: + /** Expert: implements buffer write. Writes bytes at the current position in + * the output. + * @param b the bytes to write + * @param len the number of bytes to write + */ + virtual void flushBuffer(const uint8_t* b, const int32_t len) = 0; +}; + +CL_NS_END +#endif diff --git a/3rdparty/clucene/src/CLucene/store/InputStream.h b/3rdparty/clucene/src/CLucene/store/InputStream.h new file mode 100644 index 000000000..f56819eeb --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/InputStream.h @@ -0,0 +1,21 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_InputStream_ +#define _lucene_store_InputStream_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + +deprecated... please use IndexInput.h header +and change InputStream to IndexInput + + +CL_NS_END +#endif diff --git a/3rdparty/clucene/src/CLucene/store/Lock.cpp b/3rdparty/clucene/src/CLucene/store/Lock.cpp new file mode 100644 index 000000000..a66e784b0 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/Lock.cpp @@ -0,0 +1,27 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Lock.h" + +CL_NS_DEF(store) + + bool LuceneLock::obtain(int64_t lockWaitTimeout) { + bool locked = obtain(); + int maxSleepCount = (int)(lockWaitTimeout / LOCK_POLL_INTERVAL); + int sleepCount = 0; + while (!locked) { + if (sleepCount++ == maxSleepCount) { + _CLTHROWA(CL_ERR_IO,"Lock obtain timed out"); + } + _LUCENE_SLEEP(LOCK_POLL_INTERVAL); + locked = obtain(); + } + return locked; + } + + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/Lock.h b/3rdparty/clucene/src/CLucene/store/Lock.h new file mode 100644 index 000000000..b5dda3b06 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/Lock.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_Lock_ +#define _lucene_store_Lock_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + +class LuceneLock : LUCENE_BASE +{ +public: + LUCENE_STATIC_CONSTANT(int64_t, LOCK_POLL_INTERVAL = 1000); + + virtual ~LuceneLock() {} + + // Attempts to obtain exclusive access and immediately return upon success + // or failure. Return true if exclusive access is obtained. + virtual bool obtain() = 0; + + // Attempts to obtain an exclusive lock within amount of time given. + // Currently polls once per second until lockWaitTimeout is passed. + // @param lockWaitTimeout length of time to wait in ms + // @return true if lock was obtained + // @throws IOException if lock wait times out or obtain() throws an IOException + bool obtain(int64_t lockWaitTimeout); + + // Release exclusive access. + virtual void release() = 0; + + // Returns true if the resource is currently locked. Note that one must + // still call {@link #obtain()} before using the resource. + virtual bool isLocked() = 0; + + virtual QString toString() const = 0; +}; + + +// Utility class for executing code with exclusive access. +template<typename T> +class LuceneLockWith +{ +public: + // Constructs an executor that will grab the named lock. Defaults + // lockWaitTimeout to LUCENE_COMMIT_LOCK_TIMEOUT. + // @deprecated Kept only to avoid breaking existing code. + LuceneLockWith(LuceneLock* lock, int64_t lockWaitTimeout) + { + this->lock = lock; + this->lockWaitTimeout = lockWaitTimeout; + } + + virtual ~LuceneLockWith() {} + + // Calls {@link #doBody} while <i>lock</i> is obtained. Blocks if lock + // cannot be obtained immediately. Retries to obtain lock once per second + // until it is obtained, or until it has tried ten times. Lock is released + // when {@link #doBody} exits. + T runAndReturn() + { + bool locked = false; + T ret = NULL; + try { + locked = lock->obtain(lockWaitTimeout); + ret = doBody(); + } _CLFINALLY ( + if (locked) + lock->release(); + ); + return ret; + } + + // @see runAndReturn + // Same as runAndReturn, except doesn't return any value. The only + // difference is that no void values are used + void run() + { + bool locked = false; + try { + locked = lock->obtain(lockWaitTimeout); + doBody(); + } _CLFINALLY ( + if (locked) + lock->release(); + ); + } + +protected: + virtual T doBody() = 0; + +private: + LuceneLock* lock; + int64_t lockWaitTimeout; +}; + +CL_NS_END + +#endif diff --git a/3rdparty/clucene/src/CLucene/store/MMapInput.cpp b/3rdparty/clucene/src/CLucene/store/MMapInput.cpp new file mode 100644 index 000000000..d660032c6 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/MMapInput.cpp @@ -0,0 +1,203 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#if defined(LUCENE_FS_MMAP) + +#include "FSDirectory.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +#ifndef _CLCOMPILER_MSVC + #include <sys/mman.h> +#endif + +CL_NS_DEF(store) +CL_NS_USE(util) + + FSDirectory::MMapIndexInput::MMapIndexInput(const char* path): + pos(0), + data(NULL), + _length(0), + isClone(false) + { + //Func - Constructor. + // Opens the file named path + //Pre - path != NULL + //Post - if the file could not be opened an exception is thrown. + + CND_PRECONDITION(path != NULL, "path is NULL"); + +#ifdef _CLCOMPILER_MSVC + mmaphandle = NULL; + fhandle = CreateFileA(path,GENERIC_READ,FILE_SHARE_READ, 0,OPEN_EXISTING,0,0); + + //Check if a valid fhandle was retrieved + if (fhandle < 0){ + DWORD err = GetLastError(); + if ( err == ERROR_FILE_NOT_FOUND ) + _CLTHROWA(CL_ERR_IO, "File does not exist"); + else if ( err == EACCES ) + _CLTHROWA(ERROR_ACCESS_DENIED, "File Access denied"); + else if ( err == ERROR_TOO_MANY_OPEN_FILES ) + _CLTHROWA(CL_ERR_IO, "Too many open files"); + else + _CLTHROWA(CL_ERR_IO, "File IO Error"); + } + + DWORD dummy=0; + _length = GetFileSize(fhandle,&dummy); + + if ( _length > 0 ){ + mmaphandle = CreateFileMappingA(fhandle,NULL,PAGE_READONLY,0,0,NULL); + if ( mmaphandle != NULL ){ + void* address = MapViewOfFile(mmaphandle,FILE_MAP_READ,0,0,0); + if ( address != NULL ){ + data = (uint8_t*)address; + return; //SUCCESS! + } + } + CloseHandle(mmaphandle); + + char* lpMsgBuf=0; + DWORD dw = GetLastError(); + + FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + dw, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + lpMsgBuf, + 0, NULL ); + + char* errstr = _CL_NEWARRAY(char, strlen(lpMsgBuf)+40); + sprintf(errstr, "MMapIndexInput::MMapIndexInput failed with error %d: %s", dw, lpMsgBuf); + LocalFree(lpMsgBuf); + + _CLTHROWA_DEL(CL_ERR_IO,errstr); + } + +#else //_CLCOMPILER_MSVC + fhandle = ::open (path, O_RDONLY); + if (fhandle < 0){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + // stat it + struct stat sb; + if (::fstat (fhandle, &sb)){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + // get length from stat + _length = sb.st_size; + + // mmap the file + void* address = ::mmap(0, _length, PROT_READ, MAP_SHARED, fhandle, 0); + if (address == MAP_FAILED){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + data = (uint8_t*)address; + } + } + } +#endif + } + + FSDirectory::MMapIndexInput::MMapIndexInput(const MMapIndexInput& clone): IndexInput(clone){ + //Func - Constructor + // Uses clone for its initialization + //Pre - clone is a valide instance of FSIndexInput + //Post - The instance has been created and initialized by clone + +#ifdef _CLCOMPILER_MSVC + mmaphandle = NULL; + fhandle = NULL; +#endif + + data = clone.data; + pos = clone.pos; + + //clone the file length + _length = clone._length; + //Keep in mind that this instance is a clone + isClone = true; + } + + uint8_t FSDirectory::MMapIndexInput::readByte(){ + return *(data+(pos++)); + } + + void FSDirectory::MMapIndexInput::readBytes(uint8_t* b, const int32_t len){ + memcpy(b, data+pos, len); + pos+=len; + } + int32_t FSDirectory::MMapIndexInput::readVInt(){ + uint8_t b = *(data+(pos++)); + int32_t i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = *(data+(pos++)); + i |= (b & 0x7F) << shift; + } + return i; + } + int64_t FSDirectory::MMapIndexInput::getFilePointer() const{ + return pos; + } + void FSDirectory::MMapIndexInput::seek(const int64_t pos){ + this->pos=pos; + } + + FSDirectory::MMapIndexInput::~MMapIndexInput(){ + //Func - Destructor + //Pre - True + //Post - The file for which this instance is responsible has been closed. + // The instance has been destroyed + + close(); + } + + IndexInput* FSDirectory::MMapIndexInput::clone() const + { + return _CLNEW FSDirectory::MMapIndexInput(*this); + } + void FSDirectory::MMapIndexInput::close() { + //IndexInput::close(); + + if ( !isClone ){ +#ifdef _CLCOMPILER_MSVC + if ( data != NULL ){ + if ( ! UnmapViewOfFile(data) ){ + CND_PRECONDITION( false, "UnmapViewOfFile(data) failed"); //todo: change to rich error + } + } + + if ( mmaphandle != NULL ){ + if ( ! CloseHandle(mmaphandle) ){ + CND_PRECONDITION( false, "CloseHandle(mmaphandle) failed"); + } + } + if ( fhandle != NULL ){ + if ( !CloseHandle(fhandle) ){ + CND_PRECONDITION( false, "CloseHandle(fhandle) failed"); + } + } + mmaphandle = NULL; + fhandle = NULL; +#else + if ( data != NULL ) + ::munmap(data, _length); + if ( fhandle > 0 ) + ::close(fhandle); + fhandle = 0; +#endif + } + data = NULL; + pos = 0; + } + + +CL_NS_END +#endif diff --git a/3rdparty/clucene/src/CLucene/store/OutputStream.h b/3rdparty/clucene/src/CLucene/store/OutputStream.h new file mode 100644 index 000000000..a82d6718a --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/OutputStream.h @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_IndexOutput_ +#define _lucene_store_IndexOutput_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + + +deprecated... please use IndexOutput.h header +and change OutputStream to OutdexInput + + + +CL_NS_END + +#endif // _lucene_store_IndexOutput_ diff --git a/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp b/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp new file mode 100644 index 000000000..b0a7c4d64 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp @@ -0,0 +1,446 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#include "CLucene/StdHeader.h" +#include "RAMDirectory.h" + +#include "Lock.h" +#include "Directory.h" +#include "FSDirectory.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + +RAMFile::RAMFile() +{ + length = 0; + lastModified = Misc::currentTimeMillis(); +} + +RAMFile::~RAMFile() +{ +} + + +RAMDirectory::RAMLock::RAMLock(const QString& name, RAMDirectory* dir) + : directory(dir) +{ + fname = name; +} + +RAMDirectory::RAMLock::~RAMLock() +{ + directory = NULL; +} + +QString RAMDirectory::RAMLock::toString() const +{ + return QLatin1String("LockFile@RAM"); +} + +bool RAMDirectory::RAMLock::isLocked() +{ + return directory->fileExists(fname); +} + +bool RAMDirectory::RAMLock::obtain() +{ + SCOPED_LOCK_MUTEX(directory->files_mutex); + if (!directory->fileExists(fname)) { + IndexOutput* tmp = directory->createOutput(fname); + tmp->close(); + _CLDELETE(tmp); + + return true; + } + return false; +} + +void RAMDirectory::RAMLock::release() +{ + directory->deleteFile(fname); +} + +RAMIndexOutput::~RAMIndexOutput() +{ + if (deleteFile) + _CLDELETE(file); + file = NULL; +} + +RAMIndexOutput::RAMIndexOutput(RAMFile* f) + : file(f) +{ + pointer = 0; + deleteFile = false; +} + +RAMIndexOutput::RAMIndexOutput() + : file(_CLNEW RAMFile) +{ + pointer = 0; + deleteFile = true; +} + +void RAMIndexOutput::writeTo(IndexOutput* out) +{ + flush(); + int64_t end = file->length; + int64_t pos = 0; + int32_t p = 0; + while (pos < end) { + int32_t length = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int64_t nextPos = pos + length; + if (nextPos > end) { // at the last buffer + length = (int32_t)(end - pos); + } + out->writeBytes((uint8_t*)file->buffers[p++], length); + pos = nextPos; + } +} + +void RAMIndexOutput::reset() +{ + seek(_ILONGLONG(0)); + file->length = _ILONGLONG(0); +} + +void RAMIndexOutput::flushBuffer(const uint8_t* src, const int32_t len) +{ + uint8_t* b = NULL; + int32_t bufferPos = 0; + while (bufferPos != len) { + uint32_t bufferNumber = pointer/CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bufferOffset = pointer%CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bytesInBuffer = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE - bufferOffset; + int32_t remainInSrcBuffer = len - bufferPos; + int32_t bytesToCopy = bytesInBuffer >= remainInSrcBuffer ? remainInSrcBuffer : bytesInBuffer; + + if (bufferNumber == file->buffers.size()){ + b = _CL_NEWARRAY(uint8_t, CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + file->buffers.push_back( b ); + }else{ + b = file->buffers[bufferNumber]; + } + memcpy(b+bufferOffset, src+bufferPos, bytesToCopy * sizeof(uint8_t)); + bufferPos += bytesToCopy; + pointer += bytesToCopy; + } + if (pointer > file->length) + file->length = pointer; + + file->lastModified = Misc::currentTimeMillis(); +} + +void RAMIndexOutput::close() +{ + BufferedIndexOutput::close(); +} + +/** Random-at methods */ +void RAMIndexOutput::seek(const int64_t pos) +{ + BufferedIndexOutput::seek(pos); + pointer = (int32_t)pos; +} + +int64_t RAMIndexOutput::length() +{ + return file->length; +} + + +RAMIndexInput::RAMIndexInput(RAMFile* f) + : file(f) +{ + pointer = 0; + _length = f->length; +} + +RAMIndexInput::RAMIndexInput(const RAMIndexInput& other) + : BufferedIndexInput(other) +{ + file = other.file; + pointer = other.pointer; + _length = other._length; +} + +RAMIndexInput::~RAMIndexInput() +{ + RAMIndexInput::close(); +} + +IndexInput* RAMIndexInput::clone() const +{ + return _CLNEW RAMIndexInput(*this); +} + +int64_t RAMIndexInput::length() +{ + return _length; +} + +QString RAMIndexInput::getDirectoryType() const +{ + return RAMDirectory::DirectoryType(); +} + +void RAMIndexInput::readInternal(uint8_t* dest, const int32_t len) +{ + const int64_t bytesAvailable = file->length - pointer; + int64_t remainder = len <= bytesAvailable ? len : bytesAvailable; + int32_t start = pointer; + int32_t destOffset = 0; + while (remainder != 0) { + int32_t bufferNumber = start / CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bufferOffset = start % CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bytesInBuffer = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE - bufferOffset; + + /* The buffer's entire length (bufferLength) is defined by IndexInput.h + ** as int32_t, so obviously the number of bytes in a given segment of the + ** buffer won't exceed the the capacity of int32_t. Therefore, the + ** int64_t->int32_t cast on the next line is safe. */ + int32_t bytesToCopy = bytesInBuffer >= remainder ? static_cast<int32_t>(remainder) : bytesInBuffer; + uint8_t* b = file->buffers[bufferNumber]; + memcpy(dest + destOffset, b + bufferOffset, bytesToCopy * sizeof(uint8_t)); + + destOffset += bytesToCopy; + start += bytesToCopy; + remainder -= bytesToCopy; + pointer += bytesToCopy; + } +} + +void RAMIndexInput::close() +{ + BufferedIndexInput::close(); +} + +void RAMIndexInput::seekInternal(const int64_t pos) +{ + CND_PRECONDITION(pos >= 0 && pos < this->_length, "Seeking out of range") + pointer = (int32_t)pos; +} + +// #pragma mark -- RAMDirectory + +QStringList RAMDirectory::list() const +{ + SCOPED_LOCK_MUTEX(files_mutex); + + QStringList names; + + FileMap::const_iterator itr; + for (itr = files.begin(); itr != files.end(); ++itr) + names.push_back(itr->first); + + return names; +} + +RAMDirectory::RAMDirectory() + : Directory() + , files(false, true) +{ +} + +RAMDirectory::~RAMDirectory() +{ + //todo: should call close directory? +} + +void RAMDirectory::_copyFromDir(Directory* dir, bool closeDir) +{ + QStringList names = dir->list(); + uint8_t buf[CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE]; + + foreach (const QString& name, names) { + if (!CL_NS(index)::IndexReader::isLuceneFile(name)) + continue; + + // make place on ram disk + IndexOutput* os = createOutput(name); + // read current file + IndexInput* is = dir->openInput(name); + + // and copy to ram disk + //todo: this could be a problem when copying from big indexes... + int64_t readCount = 0; + int64_t len = is->length(); + while (readCount < len) { + int32_t toRead = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + if ((readCount + toRead) > len) + toRead = int32_t(len - readCount); + is->readBytes(buf, toRead); + os->writeBytes(buf, toRead); + readCount += toRead; + } + + // graceful cleanup + is->close(); + _CLDELETE(is); + os->close(); + _CLDELETE(os); + } + if (closeDir) + dir->close(); +} + +RAMDirectory::RAMDirectory(Directory* dir) + : Directory() + , files(false, true) +{ + _copyFromDir(dir, false); +} + +RAMDirectory::RAMDirectory(const QString& dir) + : Directory() + , files(false, true) +{ + Directory* fsdir = FSDirectory::getDirectory(dir, false); + try { + _copyFromDir(fsdir, false); + } _CLFINALLY ( + fsdir->close(); + _CLDECDELETE(fsdir); + ); +} + +bool RAMDirectory::fileExists(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + return files.exists(name); +} + +int64_t RAMDirectory::fileModified(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + const RAMFile* f = files.get(name); + return f->lastModified; +} + +int64_t RAMDirectory::fileLength(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + RAMFile* f = files.get(name); + return f->length; +} + + +IndexInput* RAMDirectory::openInput(const QString& name) +{ + SCOPED_LOCK_MUTEX(files_mutex); + RAMFile* file = files.get(name); + if (file == NULL) { + _CLTHROWA(CL_ERR_IO, // DSR:PROPOSED: Better error checking. + "[RAMDirectory::open] The requested file does not exist."); + } + return _CLNEW RAMIndexInput(file); +} + +void RAMDirectory::close() +{ + SCOPED_LOCK_MUTEX(files_mutex); + files.clear(); +} + +bool RAMDirectory::doDeleteFile(const QString& name) +{ + SCOPED_LOCK_MUTEX(files_mutex); + files.remove(name); + return true; +} + +void RAMDirectory::renameFile(const QString& from, const QString& to) +{ + SCOPED_LOCK_MUTEX(files_mutex); + FileMap::iterator itr = files.find(from); + + /* DSR:CL_BUG_LEAK: + ** If a file named $to already existed, its old value was leaked. + ** My inclination would be to prevent this implicit deletion with an + ** exception, but it happens routinely in CLucene's internals (e.g., during + ** IndexWriter.addIndexes with the file named 'segments'). */ + if (files.exists(to)) + files.remove(to); + + if (itr == files.end()) { + char tmp[1024]; + _snprintf(tmp, 1024, "cannot rename %s, file does not exist", + from.toLocal8Bit().constData()); + _CLTHROWT(CL_ERR_IO, tmp); + } + + CND_PRECONDITION(itr != files.end(), "itr == files.end()") + + RAMFile* file = itr->second; + files.removeitr(itr, true, true); + files.put(to, file); +} + + +void RAMDirectory::touchFile(const QString& name) +{ + RAMFile* file = NULL; + { + SCOPED_LOCK_MUTEX(files_mutex); + file = files.get(name); + } + uint64_t ts1 = file->lastModified; + uint64_t ts2 = Misc::currentTimeMillis(); + + //make sure that the time has actually changed + while (ts1 == ts2) { + _LUCENE_SLEEP(1); + ts2 = Misc::currentTimeMillis(); + }; + + file->lastModified = ts2; +} + +IndexOutput* RAMDirectory::createOutput(const QString& name) +{ + /* Check the $files VoidMap to see if there was a previous file named + ** $name. If so, delete the old RAMFile object, but reuse the existing + ** char buffer ($n) that holds the filename. If not, duplicate the + ** supplied filename buffer ($name) and pass ownership of that memory ($n) + ** to $files. */ + + SCOPED_LOCK_MUTEX(files_mutex); + + QString n = files.getKey(name); + if (!n.isEmpty()) { + RAMFile* rf = files.get(name); + _CLDELETE(rf); + } else { + n = name; + } + + RAMFile* file = _CLNEW RAMFile(); +#ifdef _DEBUG + file->filename = n; +#endif + files[n] = file; + + return _CLNEW RAMIndexOutput(file); +} + +LuceneLock* RAMDirectory::makeLock(const QString& name) +{ + return _CLNEW RAMLock(name, this); +} + +QString RAMDirectory::toString() const +{ + return QLatin1String("RAMDirectory"); +} + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/RAMDirectory.h b/3rdparty/clucene/src/CLucene/store/RAMDirectory.h new file mode 100644 index 000000000..af92e30b2 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/RAMDirectory.h @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_RAMDirectory_ +#define _lucene_store_RAMDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "Lock.h" +#include "Directory.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Arrays.h" + +CL_NS_DEF(store) + +class RAMFile : LUCENE_BASE +{ +public: + CL_NS(util)::CLVector<uint8_t*, CL_NS(util)::Deletor::Array<uint8_t> > buffers; + int64_t length; + uint64_t lastModified; + +#ifdef _DEBUG + QString filename; +#endif + + RAMFile(); + ~RAMFile(); +}; + +class RAMIndexOutput : public BufferedIndexOutput +{ +protected: + RAMFile* file; + int32_t pointer; + bool deleteFile; + + // output methods: + void flushBuffer(const uint8_t* src, const int32_t len); + +public: + RAMIndexOutput(RAMFile* f); + RAMIndexOutput(); + /** Construct an empty output buffer. */ + virtual ~RAMIndexOutput(); + + virtual void close(); + + // Random-at methods + virtual void seek(const int64_t pos); + int64_t length(); + /** Resets this to an empty buffer. */ + void reset(); + /** Copy the current contents of this buffer to the named output. */ + void writeTo(IndexOutput* output); +}; + +class RAMIndexInput : public BufferedIndexInput +{ +private: + RAMFile* file; + int32_t pointer; + int64_t _length; + +protected: + /** IndexInput methods */ + RAMIndexInput(const RAMIndexInput& clone); + void readInternal(uint8_t *dest, const int32_t len); + + /** Random-at methods */ + void seekInternal(const int64_t pos); + +public: + RAMIndexInput(RAMFile* f); + ~RAMIndexInput(); + IndexInput* clone() const; + + void close(); + int64_t length(); + QString getDirectoryType() const; +}; + + +/** +* A memory-resident {@link Directory} implementation. +*/ +class RAMDirectory : public Directory +{ + class RAMLock : public LuceneLock + { + private: + RAMDirectory* directory; + QString fname; + public: + RAMLock(const QString& name, RAMDirectory* dir); + virtual ~RAMLock(); + bool obtain(); + void release(); + bool isLocked(); + virtual QString toString() const; + }; + + typedef CL_NS(util)::CLHashMap<QString, RAMFile*, + CL_NS(util)::Compare::Qstring, CL_NS(util)::Equals::Qstring, + CL_NS(util)::Deletor::DummyQString, + CL_NS(util)::Deletor::Object<RAMFile> > FileMap; + +protected: + /// Removes an existing file in the directory. + virtual bool doDeleteFile(const QString& name); + + /** + * Creates a new <code>RAMDirectory</code> instance from a different + * <code>Directory</code> implementation. This can be used to load + * a disk-based index into memory. + * <P> + * This should be used only with indices that can fit into memory. + * + * @param dir a <code>Directory</code> value + * @exception IOException if an error occurs + */ + void _copyFromDir(Directory* dir, bool closeDir); + FileMap files; // unlike the java Hashtable, FileMap is not synchronized, and all access must be protected by a lock + +public: +#ifndef _CL_DISABLE_MULTITHREADING //do this so that the mutable keyword still works without mt enabled + mutable DEFINE_MUTEX(files_mutex) // mutable: const methods must also be able to synchronize properly +#endif + + // Returns a null terminated array of strings, one for each file in the directory. + QStringList list() const; + + /** Constructs an empty {@link Directory}. */ + RAMDirectory(); + + // Destructor - only call this if you are sure the directory + // is not being used anymore. Otherwise use the ref-counting + // facilities of dir->close + virtual ~RAMDirectory(); + RAMDirectory(Directory* dir); + + /** + * Creates a new <code>RAMDirectory</code> instance from the {@link FSDirectory}. + * + * @param dir a <code>String</code> specifying the full index directory path + */ + RAMDirectory(const QString& dir); + + /// Returns true iff the named file exists in this directory. + bool fileExists(const QString& name) const; + + /// Returns the time the named file was last modified. + int64_t fileModified(const QString& name) const; + + /// Returns the length in bytes of a file in the directory. + int64_t fileLength(const QString& name) const; + + /// Removes an existing file in the directory. + virtual void renameFile(const QString& from, const QString& to); + + /** Set the modified time of an existing file to now. */ + void touchFile(const QString& name); + + /// Creates a new, empty file in the directory with the given name. + /// Returns a stream writing this file. + virtual IndexOutput* createOutput(const QString& name); + + /// Construct a {@link Lock}. + /// @param name the name of the lock file + LuceneLock* makeLock(const QString& name); + + /// Returns a stream reading an existing file. + IndexInput* openInput(const QString& name); + + virtual void close(); + + QString toString() const; + + static QString DirectoryType() { return QLatin1String("RAM"); } + QString getDirectoryType() const { return DirectoryType(); } +}; + +CL_NS_END + +#endif diff --git a/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp b/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp new file mode 100644 index 000000000..056fa9bc3 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "TransactionalRAMDirectory.h" + +CL_NS_DEF(store) +CL_NS_USE(util) + +TransactionalRAMDirectory::TransactionalRAMDirectory() + : RAMDirectory() + , filesToRestoreOnAbort(false, true) +{ + transOpen = false; +} + +TransactionalRAMDirectory::~TransactionalRAMDirectory() +{ +} + +bool TransactionalRAMDirectory::archiveOrigFileIfNecessary(const QString& name) +{ + // If a file named $name was present when the transaction started and the + // original RAMFile object has not been archived for restoration upon + // transaction abort, then do so, and return true. + // In any other case, return false. + if (fileExists(name) && filesToRemoveOnAbort.find(name) == filesToRemoveOnAbort.end()) { + // The file exists, but isn't recorded as having been created after the + // start of the transaction, so it must've been present at the start of + // the transaction. + + // Transfer memory ownership of both the key and the value from files to + // filesToRestoreOnAbort. + QString origName = files.getKey(name); + RAMFile* origFile = files.get(name); + files.remove(name, true, true); + filesToRestoreOnAbort.put(origName, origFile); + + CND_CONDITION(!fileExists(name), + "File should not exist immediately after archival."); + return true; + } + + return false; +} + +void TransactionalRAMDirectory::unarchiveOrigFile(const QString& name) +{ + QString origName = filesToRestoreOnAbort.getKey(name); + if (origName.isEmpty()) { + _CLTHROWA(CL_ERR_RAMTransaction, + "File submitted for unarchival was not archived."); + } + RAMFile* origFile = filesToRestoreOnAbort.get(name); + // Transfer memory ownership back to files from filesToRestoreOnAbort. + filesToRestoreOnAbort.remove(name, true, true); + files.put(origName, origFile); +} + +bool TransactionalRAMDirectory::transIsOpen() const +{ + return transOpen; +} + +void TransactionalRAMDirectory::transStart() +{ + if (transOpen) { + _CLTHROWA(CL_ERR_RAMTransaction, + "Must resolve previous transaction before starting another."); + } + + CND_CONDITION(filesToRemoveOnAbort.size() == 0, + "filesToRemoveOnAbort should have been cleared by either its" + " constructor or transResolved."); + + CND_CONDITION(filesToRestoreOnAbort.size() == 0, + "filesToRestoreOnAbort should have been cleared by either its" + " constructor or transResolved."); + + transOpen = true; +} + +void TransactionalRAMDirectory::transResolved() +{ + // This method implements actions common to both forms of transaction + // resolution. + filesToRemoveOnAbort.clear(); + filesToRestoreOnAbort.clear(); + transOpen = false; +} + +void TransactionalRAMDirectory::transCommit() +{ + if (!transOpen) + _CLTHROWA(CL_ERR_RAMTransaction, "There is no open transaction."); + + // All storage is in memory, so commit is ultra-simple. + transResolved(); +} + +void TransactionalRAMDirectory::transAbort() +{ + if (!transOpen) + _CLTHROWA(CL_ERR_RAMTransaction, "There is no open transaction."); + + // Delete each file in filesToRemoveOnAbort. + FilenameSet::const_iterator itrDel = filesToRemoveOnAbort.begin(); + for ( ; itrDel != filesToRemoveOnAbort.end(); ++itrDel) { + size_t nameLength = itrDel->first.length(); + + // Special exception: Refrain from deleting a lock's flag file, as that + // would interfere with the operation of the lock. + if (!(nameLength >= 5 + && itrDel->first.rightRef(5) == QLatin1String(".lock"))) { + RAMDirectory::deleteFile(itrDel->first); + } + } + // Ownership of the memory of both the key and the value never left files, + // so there's no need for a special directive to filesToRemoveOnAbort. + filesToRemoveOnAbort.clear(); + + // Now that any new-since-trans-start files with the same names as + // already-present-at-trans-start files are out of the way, restore each + // file in filesToRestoreOnAbort. + TransFileMap::const_iterator itr = filesToRestoreOnAbort.begin(); + for ( ; itr != filesToRestoreOnAbort.end(); ++itr) { + files.put(itr->first, itr->second); + filesToRestoreOnAbort.remove(itr->first); + } + + CND_CONDITION(filesToRestoreOnAbort.size() == 0, + "filesToRestoreOnAbort should be empty."); + + transResolved(); +} + +bool TransactionalRAMDirectory::doDeleteFile(const QString& name) +{ + if (!transOpen) + return RAMDirectory::doDeleteFile(name); + + bool wasOriginalAndWasArchived = archiveOrigFileIfNecessary(name); + if (!wasOriginalAndWasArchived) { + // The file to be deleted wasn't present at transaction start, so instead + // of archiving it, we delete it the conventional way, making sure to + // erase its record in filesToRemoveOnAbort if it was listed there. + filesToRemoveOnAbort.remove(name); + return RAMDirectory::doDeleteFile(name); + } + return true; +} + +void TransactionalRAMDirectory::renameFile(const QString& from, const QString& to) +{ + // During the review on 2005.03.18, decided not to implement transactional + // renameFile for two reasons: + // a) It's not needed in the limited scenario for which + // TransactionalRAMDirectory was designed (IndexWriter::addDocument and + // subcode). + // b) Supporting renaming during a transaction would add considerable + // bookkeeping overhead, reducing the performance of the overwhelmingly + // typical case (commit) in order to support the rare case (abort). + // + // This was not a thinly disguised punt due to the complication of + // implementing renameFile transactionally; rather, several implementations + // were considered, but it seemed wrongheaded to degrade the performance of + // the typical case based on the mere potential need to support renameFile + // at some future point for the benefit of the atypical case. + if (transOpen) { + _CLTHROWA(CL_ERR_RAMTransaction, + "TransactionalRAMDirectory disallows renameFile during a transaction."); + } + RAMDirectory::renameFile(from, to); +} + +IndexOutput* TransactionalRAMDirectory::createOutput(const QString& name) +{ + if (!transOpen) + return RAMDirectory::createOutput(name); + + bool wasOriginalAndWasArchived = archiveOrigFileIfNecessary(name); + try { + IndexOutput* ret = RAMDirectory::createOutput(name); + // Importantly, we store a pointer to the filename memory managed by + // files, rather than that passed in by the client (name). We don't make + // an additional copy of the filename's memory because the transactional + // metadata container filesToRemoveOnAbort is not at risk of outliving + // files. + filesToRemoveOnAbort.put(files.getKey(name), NULL); + return ret; + } catch (...) { + if (wasOriginalAndWasArchived) { + unarchiveOrigFile(name); + } + throw; + } +} + +void TransactionalRAMDirectory::close() +{ + if (transOpen) + transAbort(); + + RAMDirectory::close(); +} + +CL_NS_END diff --git a/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h b/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h new file mode 100644 index 000000000..44c5e8e99 --- /dev/null +++ b/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_TransactionalRAMDirectory_ +#define _lucene_store_TransactionalRAMDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "RAMDirectory.h" +#include "CLucene/util/VoidList.h" + +CL_NS_DEF(store) + +/*** +This transactional in-memory Directory was created to address a specific +situation, and was deliberately pared down to the simplest viable +implementation. For the sake of simplicity, this implementation imposes +restrictions on what operations can be performed in the directory while a +transaction is in progress (documented in TransactionalRAMDirectory.cpp). + +Because the Lucene Directory interface itself is rather simplistic, it +would not be difficult to expand TransactionalRAMDirectory so that it +provided fully general transactionality. However, the developer of this +original implementation was of the opinion that the last thing CLucene +needs is gratuitous features that exceed their required complexity and +haven't been rigorously tested. +*/ +class TransactionalRAMDirectory : public RAMDirectory +{ +private: + typedef CL_NS(util)::CLSet<QString, void*, CL_NS(util)::Compare::Qstring, + CL_NS(util)::Deletor::DummyQString> FilenameSet; + FilenameSet filesToRemoveOnAbort; + + typedef CL_NS(util)::CLSet<QString, RAMFile*, CL_NS(util)::Compare::Qstring, + CL_NS(util)::Deletor::DummyQString, + CL_NS(util)::Deletor::Object<RAMFile> > TransFileMap; + TransFileMap filesToRestoreOnAbort; + + bool transOpen; + + void transResolved(); + bool archiveOrigFileIfNecessary(const QString& name); + void unarchiveOrigFile(const QString& name); + +protected: + bool doDeleteFile(const QString& name); + +public: + TransactionalRAMDirectory(); + virtual ~TransactionalRAMDirectory(); + + bool transIsOpen() const; + void transStart(); + void transCommit(); + void transAbort(); + + // Constrained operations: + void renameFile(const QString& from, const QString& to); + IndexOutput* createOutput(const QString& name); + + void close(); +}; + +CL_NS_END + +#endif |