/*
* tests/persist/cog/LargeZipfTest.cxxtest
*
* Emulate word-pair dataset with Zipfian incoming sets.
* This creates nodes with large incoming sets; database should
* not choke on these.
*
* Copyright (C) 2008, 2009, 2013, 2019 Linas Vepstas <linasvepstas@gmail.com>
* SPDX-License-Identifier: AGPL-3.0-or-later
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License v3 as
* published by the Free Software Foundation and including the exceptions
* at http://opencog.org/wiki/Licenses
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program; if not, write to:
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include <cstdio>
#include <opencog/atoms/atom_types/atom_types.h>
#include <opencog/persist/cog-types/atom_types.h>
#include <opencog/atoms/base/Atom.h>
#include <opencog/atoms/base/Link.h>
#include <opencog/atoms/base/Node.h>
#include <opencog/atoms/value/FloatValue.h>
#include <opencog/atoms/value/VoidValue.h>
#include <opencog/persist/cog-storage/CogStorage.h>
#include "../TestCogServer.h"
#include <opencog/util/Logger.h>
using namespace opencog;
class LargeZipfUTest : public CxxTest::TestSuite
{
private:
AtomSpacePtr _as;
DECLARE_TEST_COGSERVER
#define NWORDS 35000
Handle hword[NWORDS];
#define NPAIRS 257000
Handle hpair[NPAIRS];
int npairs; // actual number of pairs created.
public:
LargeZipfUTest(void);
~LargeZipfUTest();
void setUp(void);
void tearDown(void);
void kill_data(void);
int filler_up(AtomSpacePtr);
void fetch_space(AtomSpacePtr, StorageNodePtr);
void check_space(AtomSpacePtr, std::string dbgmsg);
void check_empty(AtomSpacePtr);
void atomCompare(Handle, AtomPtr, std::string);
void test_atomspace(void);
};
LargeZipfUTest:: LargeZipfUTest(void)
{
// logger().set_level(Logger::DEBUG);
logger().set_level(Logger::INFO);
logger().set_print_to_stdout_flag(true);
INIT_TEST_COGSERVER(16009);
printf("Started CogServer\n");
}
LargeZipfUTest::~LargeZipfUTest()
{
STOP_TEST_COGSERVER
logger().flush();
// erase the log file if no assertions failed
if (!CxxTest::TestTracker::tracker().suiteFailed())
std::remove(logger().get_filename().c_str());
}
/*
* This is called once before each test, for each test (!!)
*/
void LargeZipfUTest::setUp(void)
{
_as = createAtomSpace();
}
void LargeZipfUTest::tearDown(void)
{
_as = nullptr;
kill_data();
}
// ============================================================
void LargeZipfUTest::kill_data(void)
{
CogStorage *astore = new CogStorage("cog://localhost:16009");
astore->open();
if (!astore->connected())
{
logger().info("setUp: CogStorage cannot connect to server");
exit(1);
}
// Selectively clear test data (preserves CogServerNode)
::kill_data(astore, _test_asp.get());
// Destructor also logs out
delete astore;
}
// ============================================================
void LargeZipfUTest::atomCompare(Handle a, AtomPtr b, std::string where)
{
// printf("Check %s expect %s\n", where.c_str(), a->to_string().c_str());
TSM_ASSERT("No atom found", b != nullptr);
if (nullptr == b) return;
TSM_ASSERT_EQUALS("Type mismatch", a->get_type(), b->get_type());
if (a->is_node())
{
TSM_ASSERT_EQUALS("Name mismatch", a->get_name(), b->get_name());
}
if (a->is_link())
{
TSM_ASSERT_EQUALS("Arity mismatch", a->get_arity(), b->get_arity());
if (*a != *b)
{
printf("Mismatching la = %s\n", a->to_string().c_str());
printf("Mismatching lb = %s\n", b->to_string().c_str());
TSM_ASSERT_EQUALS("Link mis-match", a, b);
}
}
ValuePtr ta = a->getValue(truth_key());
ValuePtr tb = b->getValue(truth_key());
if (ta or tb)
{
TSM_ASSERT("Missing truth value", ta);
TSM_ASSERT("Missing truth value", tb);
if (ta and tb)
{
TSM_ASSERT("Truth value miscompare", (*ta)==(*tb));
if (not ((*ta) == (*tb)))
{
FloatValuePtr fva = FloatValueCast(ta);
FloatValuePtr fvb = FloatValueCast(tb);
fprintf(stderr, "Error, truth value miscompare, "
"ma=%f mb=%f ca=%f cb=%f\n",
fva->value()[0], fvb->value()[0],
fva->value()[1], fvb->value()[1]);
}
}
}
}
// ============================================================
void LargeZipfUTest::fetch_space(AtomSpacePtr space, StorageNodePtr store)
{
for (int w=0; w<NWORDS; w++)
{
std::string nam = hword[w]->get_name();
Handle h = space->add_node(hword[w]->get_type(), std::move(nam));
store->fetch_atom(h); // get the TV
}
for (int p=0; p<npairs; p++)
{
HandleSeq hs = hpair[p]->getOutgoingSet();
Handle h = space->add_link(hpair[p]->get_type(), std::move(hs));
store->fetch_atom(h); // get the TV
}
}
void LargeZipfUTest::check_space(AtomSpacePtr space, std::string dbgmsg)
{
for (int w=0; w<NWORDS; w++)
{
std::string nam = hword[w]->get_name();
Handle h = space->get_node(hword[w]->get_type(), std::move(nam));
atomCompare(hword[w], h, dbgmsg + "-word-" + std::to_string(w));
}
for (int p=0; p<npairs; p++)
{
HandleSeq hs = hpair[p]->getOutgoingSet();
Handle h = space->get_link(hpair[p]->get_type(), std::move(hs));
atomCompare(hpair[p], h, dbgmsg + "pair-" + std::to_string(p));
}
}
void LargeZipfUTest::check_empty(AtomSpacePtr space)
{
for (int w=0; w<NWORDS; w++)
{
std::string nam = hword[w]->get_name();
Handle h = space->get_node(hword[w]->get_type(), std::move(nam));
TSM_ASSERT("Should not find this word", h == Handle::UNDEFINED);
}
for (int p=0; p<npairs; p++)
{
HandleSeq hs = hpair[p]->getOutgoingSet();
Handle h = space->get_link(hpair[p]->get_type(), std::move(hs));
TSM_ASSERT("Should not find this pair", h == Handle::UNDEFINED);
}
}
// ============================================================
int LargeZipfUTest::filler_up(AtomSpacePtr space)
{
// Emulate a word, with a spelling that is not long, not short ...
std::string wrd = "Word-ishy ";
for (int w=0; w<NWORDS; w++)
{
hword[w] = space->add_node(CONCEPT_NODE, wrd + std::to_string(w));
ValuePtr tv(createFloatValue(std::vector<double>({1, 0, (double)((int) NWORDS/(w+1))})));
hword[w]->setValue(truth_key(), tv);
}
// Emulate a zipfian distribution.
// Half of words linked once. rpt = 1, wmax = NWORDS
// quarter of words linked twice. rpt = 2, wmax = NWORDS/2
// eighth of words linked 4 times. rpt = 4, wmax = NWORDS/4
// 1/16 of words linked 8 times. rpt = 8, wmax = NWORDS/8
int w1 = 0;
int w2 = 0;
int wmax = NWORDS;
int rpt = 1;
int again = 0;
int p = 0;
while (p<NPAIRS)
{
hpair[p] = space->add_link(LIST_LINK, hword[w1], hword[w2]);
ValuePtr tv(createFloatValue(std::vector<double>({1, 0, (double)wmax})));
hpair[p]->setValue(truth_key(), tv);
w2++;
p++;
if (wmax <= w2)
{
w2=0;
w1++;
again ++;
if (rpt <= again)
{
again = 0;
rpt *= 2;
}
wmax = NWORDS / (double) (w1 + 1);
if (NWORDS <= w1) break;
}
}
npairs = p;
logger().info("Created %d words and %d pairs", NWORDS, npairs);
for (int i=0; i<20; i++)
{
logger().info("Word %d has incoming size=%zu", i,
hword[i]->getIncomingSetSize());
}
return p;
}
// ============================================================
void LargeZipfUTest::test_atomspace(void)
{
logger().debug("BEGIN TEST: %s", __FUNCTION__);
Handle hsn = _as->add_node(COG_STORAGE_NODE, "cog://localhost:16009");
StorageNodePtr store = StorageNodeCast(hsn);
store->setValue(_as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
TS_ASSERT(store->connected())
/* Create a large number of atoms */
filler_up(_as);
/* Verify that the atoms were added */
check_space(_as, "verify-add");
/* Push all atoms out to the SQL DB */
store->store_atomspace();
/* Extract atoms from the AtomSpace. This does not delete them from
* the SQL storage, though; to do that, they must be deleted, not
* extracted.
*/
store->barrier();
_as->clear();
TSM_ASSERT("Non-empty atomspace", 0 == _as->get_size());
/* The clear above nuked the storage, so restart */
hsn = _as->add_node(COG_STORAGE_NODE, "cog://localhost:16009");
store = StorageNodeCast(hsn);
store->setValue(_as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
TS_ASSERT(store->connected())
/* Verify that the atoms can still be fetched from storage. */
fetch_space(_as, store);
store->barrier();
check_space(_as, "verify-fetch");
/* Do it again, for good luck. */
_as->clear();
TSM_ASSERT("Non-empty atomspace", 0 == _as->get_size());
/* The clear above nuked the storage, so restart */
hsn = _as->add_node(COG_STORAGE_NODE, "cog://localhost:16009");
store = StorageNodeCast(hsn);
store->setValue(_as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
TS_ASSERT(store->connected())
/* Verify that the atoms can still be fetched from storage. */
fetch_space(_as, store);
store->barrier();
check_space(_as, "ver-again");
/* Kill data for good */
_as->clear();
TSM_ASSERT("Non-empty atomspace", 0 == _as->get_size());
/* Verify that the atoms are no longer being found. */
check_empty(_as);
store->setValue(_as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
logger().debug("END TEST: %s", __FUNCTION__);
}
/* ============================= END OF FILE ================= */