blob: 6cd50b20224da59fb5dae6899b0bfb992874d710 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause
#include <QList>
#include <QMap>
#include <QTextStream>
#include <QString>
#include <QStringList>
#include <QDir>
#include <QElapsedTimer>
#include <QApplication>
#include <QDebug>
#include <qtconcurrentmap.h>
using namespace QtConcurrent;
/*
Utility function that recursivily searches for files.
*/
QStringList findFiles(const QString &startDir, const QStringList &filters)
{
QStringList names;
QDir dir(startDir);
const auto files = dir.entryList(filters, QDir::Files);
for (const QString &file : files)
names += startDir + '/' + file;
const auto subdirs = dir.entryList(QDir::AllDirs | QDir::NoDotAndDotDot);
for (const QString &subdir : subdirs)
names += findFiles(startDir + '/' + subdir, filters);
return names;
}
typedef QMap<QString, int> WordCount;
/*
Single threaded word counter function.
*/
WordCount singleThreadedWordCount(const QStringList &files)
{
WordCount wordCount;
for (const QString &file : files) {
QFile f(file);
f.open(QIODevice::ReadOnly);
QTextStream textStream(&f);
while (!textStream.atEnd()) {
const auto words = textStream.readLine().split(' ');
for (const QString &word : words)
wordCount[word] += 1;
}
}
return wordCount;
}
// countWords counts the words in a single file. This function is
// called in parallel by several threads and must be thread
// safe.
WordCount countWords(const QString &file)
{
QFile f(file);
f.open(QIODevice::ReadOnly);
QTextStream textStream(&f);
WordCount wordCount;
while (!textStream.atEnd()) {
const auto words = textStream.readLine().split(' ');
for (const QString &word : words)
wordCount[word] += 1;
}
return wordCount;
}
// reduce adds the results from map to the final
// result. This functor will only be called by one thread
// at a time.
void reduce(WordCount &result, const WordCount &w)
{
for (auto i = w.begin(), end = w.end(); i != end; ++i)
result[i.key()] += i.value();
}
int main(int argc, char** argv)
{
QApplication app(argc, argv);
qDebug() << "finding files...";
QStringList files = findFiles("../../", QStringList() << "*.cpp" << "*.h");
qDebug() << files.count() << "files";
qDebug() << "warmup";
{
WordCount total = singleThreadedWordCount(files);
}
qDebug() << "warmup done";
int singleThreadTime = 0;
{
QElapsedTimer timer;
timer.start();
WordCount total = singleThreadedWordCount(files);
singleThreadTime = timer.elapsed();
qDebug() << "single thread" << singleThreadTime;
}
int mapReduceTime = 0;
{
QElapsedTimer timer;
timer.start();
WordCount total = mappedReduced(files, countWords, reduce).result();
mapReduceTime = timer.elapsed();
qDebug() << "MapReduce" << mapReduceTime;
}
qDebug() << "MapReduce speedup x" << ((double)singleThreadTime - (double)mapReduceTime) / (double)mapReduceTime + 1;
}
|
