OverviewFilesCommitsIssuesDocsReleasesPeers

opencog-SimpleGPUHashTable commits / 4548ce8

Added test.cpp
4548ce812e7ed151d5adf215d1ceac15a6cf1f06
by David Farrell <nosferalatu@gmail.com> · 2020-03-02T00:57:02.000Z

Changed files (4) +158 -141

CMakeLists.txt +1 -1
……@@ -7,7 +7,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
77
88string(APPEND CMAKE_CUDA_FLAGS " -gencode arch=compute_35,code=sm_35")
99
10set(SOURCE_FILES src/test.cpp src/linearprobing.h src/linearprobing.cu)
10set(SOURCE_FILES src/main.cpp src/test.cpp src/linearprobing.h src/linearprobing.cu)
1111
1212include_directories(${CMAKE_SOURCE_DIR}/src)
1313
src/linearprobing.cu +2 -1
……@@ -79,7 +79,8 @@ void insert_hashtable(KeyValue* pHashTable, const KeyValue* kvs, uint32_t num_kv
7979 float milliseconds = 0;
8080 cudaEventElapsedTime(&milliseconds, start, stop);
8181 float seconds = milliseconds / 1000.0f;
82 printf(" GPU inserted %d items in %f ms (%f million keys/second)\n", num_kvs, milliseconds, num_kvs / (double)seconds / 1000000.0f);
82 printf(" GPU inserted %d items in %f ms (%f million keys/second)\n",
83 num_kvs, milliseconds, num_kvs / (double)seconds / 1000000.0f);
8384
8485 cudaFree(device_kvs);
8586}
src/main.cpp +110 -0
……@@ -0,0 +1,110 @@
1#include "windows.h"
2#include "stdio.h"
3#include "stdint.h"
4#include "unordered_set"
5#include "unordered_map"
6#include "vector"
7#include "algorithm"
8#include "random"
9#include "linearprobing.h"
10
11std::vector<KeyValue> generate_random_keyvalues()
12{
13 std::random_device rd;
14 std::mt19937 gen(rd()); // mersenne_twister_engine
15 std::uniform_int_distribution<uint32_t> dis(1, 0xffffffff); // 0 is reserved for empty keys
16
17 std::vector<KeyValue> kvs;
18 kvs.reserve(kNumKeyValues);
19
20 for (int i = 0; i < kNumKeyValues; i++)
21 {
22 uint32_t rand0 = dis(gen);
23 uint32_t rand1 = dis(gen);
24 kvs.push_back(KeyValue{ rand0, rand1 });
25 }
26
27 return kvs;
28}
29
30LARGE_INTEGER start_timer()
31{
32 LARGE_INTEGER timer;
33 QueryPerformanceCounter(&timer);
34 return timer;
35}
36
37double get_elapsed_time(LARGE_INTEGER start)
38{
39 LARGE_INTEGER end;
40 QueryPerformanceCounter(&end);
41
42 LARGE_INTEGER frequency;
43 QueryPerformanceFrequency(&frequency);
44
45 LARGE_INTEGER elapsed;
46 elapsed.QuadPart = end.QuadPart - start.QuadPart;
47 elapsed.QuadPart *= 1000000;
48 double milliseconds = elapsed.QuadPart / (double)frequency.QuadPart / 1000;
49
50 return milliseconds;
51}
52
53void test_unordered_map(std::vector<KeyValue> all_kvs)
54{
55 LARGE_INTEGER timer = start_timer();
56
57 std::unordered_map<uint32_t, uint32_t> kvs_map;
58 for (auto& kv : all_kvs)
59 {
60 kvs_map[kv.key] = kv.value;
61 }
62
63 double milliseconds = get_elapsed_time(timer);
64 double seconds = milliseconds / 1000.0f;
65 printf("std::unordered_map time : %f ms (%f million keys/second)\n",
66 milliseconds, kNumKeyValues / seconds / 1000000.0f);
67}
68
69void test_correctness(std::vector<KeyValue>, std::vector<KeyValue>);
70
71int main()
72{
73 printf("Testing insertion of %d elements\n", kNumKeyValues);
74
75 // Initialize keyvalue pairs with random numbers
76 std::vector<KeyValue> all_kvs = generate_random_keyvalues();
77
78 // Begin test
79 LARGE_INTEGER timer = start_timer();
80
81 KeyValue* pHashTable = create_hashtable(kHashTableCapacity);
82
83 // Insert all the elements; each batch is processed concurrently on the GPU
84 const uint32_t num_batch_insertions = 16;
85 static_assert(kNumKeyValues % num_batch_insertions == 0, "needs even divisor");
86 uint32_t num_kvs_in_batch = kNumKeyValues / num_batch_insertions;
87 for (int i = 0; i < num_batch_insertions; i++)
88 {
89 insert_hashtable(pHashTable, all_kvs.data() + i*num_kvs_in_batch, num_kvs_in_batch);
90 }
91
92 // Get all the key-values from the hash table
93 std::vector<KeyValue> kvs = iterate_hashtable(pHashTable);
94
95 destroy_hashtable(pHashTable);
96
97 // Summarize results
98 double milliseconds = get_elapsed_time(timer);
99 double seconds = milliseconds / 1000.0f;
100 printf("Total time (including memory copies, readback, etc): %f ms (%f million keys/second)\n",
101 milliseconds, kNumKeyValues / seconds / 1000000.0f);
102
103 test_unordered_map(all_kvs);
104
105 test_correctness(all_kvs, kvs);
106
107 printf("Success\n");
108
109 return 0;
110}
src/test.cpp +45 -139
……@@ -8,164 +8,70 @@
88#include "random"
99#include "linearprobing.h"
1010
11std::vector<KeyValue> generate_random_keyvalues()
11void test_correctness(std::vector<KeyValue> all_kvs, std::vector<KeyValue> kvs)
1212{
13 std::random_device rd;
14 std::mt19937 gen(rd()); // mersenne_twister_engine
15 std::uniform_int_distribution<uint32_t> dis(1, 0xffffffff); // 0 is reserved for empty keys
16
17 std::vector<KeyValue> kvs;
18 kvs.reserve(kNumKeyValues);
19
20 for (int i = 0; i < kNumKeyValues; i++)
13 printf("Testing that there are no duplicate keys...\n");
14 std::unordered_set<uint32_t> unique_keys;
15 for (uint32_t i = 0; i < kvs.size(); i++)
2116 {
22 uint32_t rand0 = dis(gen);
23 uint32_t rand1 = dis(gen);
24 kvs.push_back(KeyValue{ rand0, rand1 });
25 }
26
27 return kvs;
28}
29
30LARGE_INTEGER start_timer()
31{
32 LARGE_INTEGER timer;
33 QueryPerformanceCounter(&timer);
34 return timer;
35}
36
37double get_elapsed_time(LARGE_INTEGER start)
38{
39 LARGE_INTEGER end;
40 QueryPerformanceCounter(&end);
41
42 LARGE_INTEGER frequency;
43 QueryPerformanceFrequency(&frequency);
44
45 LARGE_INTEGER elapsed;
46 elapsed.QuadPart = end.QuadPart - start.QuadPart;
47 elapsed.QuadPart *= 1000000;
48 double milliseconds = elapsed.QuadPart / (double)frequency.QuadPart / 1000;
17 if (i % 10000000 == 0)
18 printf(" Verifying %d/%d\n", i, (uint32_t)kvs.size());
4919
50 return milliseconds;
51}
52
53void test_unordered_map(std::vector<KeyValue> all_kvs)
54{
55 LARGE_INTEGER timer = start_timer();
56
57 std::unordered_map<uint32_t, uint32_t> kvs_map;
58 for (auto& kv : all_kvs)
59 {
60 kvs_map[kv.key] = kv.value;
20 KeyValue* node = &kvs[i];
21 if (unique_keys.find(node->key) != unique_keys.end())
22 {
23 printf("Duplicate key found in GPU hash table at slot %d\n", i);
24 exit(-1);
25 }
26 unique_keys.insert(node->key);
6127 }
6228
63 double milliseconds = get_elapsed_time(timer);
64 double seconds = milliseconds / 1000.0f;
65 printf("std::unordered_map time : %f ms (%f million keys/second)\n", milliseconds, kNumKeyValues / seconds / 1000000.0f);
66}
67
68int main()
69{
70 printf("Testing insertion of %d elements\n", kNumKeyValues);
71
72 // Initialize keyvalue pairs with random numbers
73 std::vector<KeyValue> all_kvs = generate_random_keyvalues();
74
75 // Begin test
76 LARGE_INTEGER timer = start_timer();
77
78 KeyValue* pHashTable = create_hashtable(kHashTableCapacity);
79
80 // Insert all the elements; each batch is processed concurrently on the GPU
81 const uint32_t num_batch_insertions = 16;
82 static_assert(kNumKeyValues % num_batch_insertions == 0, "needs even divisor");
83 uint32_t num_kvs_in_batch = kNumKeyValues / num_batch_insertions;
84 for (int i = 0; i < num_batch_insertions; i++)
29 printf("Building unordered_map from insertion list...\n");
30 std::unordered_map<uint32_t, std::vector<uint32_t>> all_kvs_map;
31 for (int i = 0; i < all_kvs.size(); i++)
8532 {
86 insert_hashtable(pHashTable, all_kvs.data() + i*num_kvs_in_batch, num_kvs_in_batch);
87 }
88
89 // Get all the key-values from the hash table
90 std::vector<KeyValue> kvs = iterate_hashtable(pHashTable);
33 if (i % 10000000 == 0)
34 printf(" Building %d/%d\n", i, (uint32_t)all_kvs.size());
9135
92 destroy_hashtable(pHashTable);
93
94 // Summarize results
95 double milliseconds = get_elapsed_time(timer);
96 double seconds = milliseconds / 1000.0f;
97 printf("Total time (including memory copies, readback, etc.): %f ms (%f million keys/second)\n", milliseconds, kNumKeyValues / seconds / 1000000.0f);
98
99 test_unordered_map(all_kvs);
100
101 // Test the results
102 if (true)
103 {
104 printf("Testing that there are no duplicate keys...\n");
105 std::unordered_set<uint32_t> unique_keys;
106 for (uint32_t i = 0; i < kvs.size(); i++)
36 auto iter = all_kvs_map.find(all_kvs[i].key);
37 if (iter == all_kvs_map.end())
10738 {
108 if (i % 10000000 == 0)
109 printf(" Verifying %d/%d\n", i, (uint32_t)kvs.size());
110
111 KeyValue* node = &kvs[i];
112 if (unique_keys.find(node->key) != unique_keys.end())
113 {
114 printf("Duplicate key found in GPU hash table at slot %d\n", i);
115 exit(-1);
116 }
117 unique_keys.insert(node->key);
39 all_kvs_map[all_kvs[i].key] = std::vector<uint32_t>({ all_kvs[i].value });
11840 }
119
120 printf("Building unordered_map from insertion list...\n");
121 std::unordered_map<uint32_t, std::vector<uint32_t>> all_kvs_map;
122 for (int i = 0; i < all_kvs.size(); i++)
41 else
12342 {
124 if (i % 10000000 == 0)
125 printf(" Building %d/%d\n", i, (uint32_t)all_kvs.size());
126
127 auto iter = all_kvs_map.find(all_kvs[i].key);
128 if (iter == all_kvs_map.end())
129 {
130 all_kvs_map[all_kvs[i].key] = std::vector<uint32_t>({ all_kvs[i].value });
131 }
132 else
133 {
134 iter->second.push_back(all_kvs[i].value);
135 }
43 iter->second.push_back(all_kvs[i].value);
13644 }
45 }
46
47 if (unique_keys.size() != all_kvs_map.size())
48 {
49 printf("# of unique keys in hashtable is incorrect\n");
50 exit(-1);
51 }
13752
138 if (unique_keys.size() != all_kvs_map.size())
53 printf("Testing that each key/value in hashtable is in the insertion list...\n");
54 for (uint32_t i = 0; i < kvs.size(); i++)
55 {
56 if (i % 10000000 == 0)
57 printf(" Verifying %d/%d\n", i, (uint32_t)kvs.size());
58
59 auto iter = all_kvs_map.find(kvs[i].key);
60 if (iter == all_kvs_map.end())
13961 {
140 printf("# of unique keys in hashtable is incorrect\n");
62 printf("Hashtable key not found in insertion list\n");
14163 exit(-1);
14264 }
14365
144 printf("Testing that each key/value in hashtable is in the insertion list...\n");
145 for (uint32_t i = 0; i < kvs.size(); i++)
66 std::vector<uint32_t>& values = iter->second;
67 if (std::find(values.begin(), values.end(), kvs[i].value) == values.end())
14668 {
147 if (i % 10000000 == 0)
148 printf(" Verifying %d/%d\n", i, (uint32_t)kvs.size());
149
150 auto iter = all_kvs_map.find(kvs[i].key);
151 if (iter == all_kvs_map.end())
152 {
153 printf("Hashtable key not found in insertion list\n");
154 exit(-1);
155 }
156
157 std::vector<uint32_t>& values = iter->second;
158 if (std::find(values.begin(), values.end(), kvs[i].value) == values.end())
159 {
160 printf("Hashtable value not found in insertion list\n");
161 exit(-1);
162 }
69 printf("Hashtable value not found in insertion list\n");
70 exit(-1);
16371 }
164
165 printf("Deleting std::unordered_map and std::unique_set...\n");
16672 }
16773
168 printf("Success\n");
74 printf("Deleting std::unordered_map and std::unique_set...\n");
16975
170 return 0;
76 return;
17177}