OverviewFilesCommitsIssuesDocsReleasesPeers

opencog-atomspace-cog / tests / persist / cog-storage / BasicDeleteUTest.cxxtest

509 lines · cxxtest ref: 
/*
 * tests/persist/cog-storage/BasicDeleteUTest.cxxtest
 *
 * Verify that deletion of keys works, even if some Atoms are still
 * using those keys.
 *
 * Copyright (C) 2008, 2009, 2019, 2021 Linas Vepstas <linasvepstas@gmail.com>
 *
 * LICENSE:
 * SPDX-License-Identifier: AGPL-3.0-or-later
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#include <cstdio>

#include <opencog/atoms/base/Atom.h>
#include <opencog/atoms/base/Link.h>
#include <opencog/atoms/base/Node.h>
#include <opencog/atoms/atom_types/atom_types.h>
#include <opencog/persist/cog-types/atom_types.h>
#include <opencog/atoms/value/FloatValue.h>
#include <opencog/atoms/value/LinkValue.h>
#include <opencog/atoms/value/VoidValue.h>
#include <opencog/atomspace/AtomSpace.h>
#include "../TestCogServer.h"

#include <opencog/persist/api/StorageNode.h>

#include <opencog/util/Logger.h>

using namespace opencog;

class BasicDeleteUTest :  public CxxTest::TestSuite
{
	private:
		std::string uri;
		DECLARE_TEST_COGSERVER

	public:

		BasicDeleteUTest(void)
		{
			logger().set_level(Logger::INFO);
			// logger().set_level(Logger::DEBUG);
			logger().set_print_to_stdout_flag(true);

			uri = "cog://localhost:16010";

			INIT_TEST_COGSERVER(16010);
			printf("Started CogServer\n");
		}

		~BasicDeleteUTest()
		{
			STOP_TEST_COGSERVER

			// erase the log file if no assertions failed
			if (!CxxTest::TestTracker::tracker().suiteFailed())
				std::remove(logger().get_filename().c_str());
		}

		void setUp(void);
		void tearDown(void);

		void test_duplicates(void);
		void test_recursive(void);
		void test_key_deletion(void);
		void test_key_corruption(void);
};

/*
 * This is called once before each test, for each test (!!)
 */
void BasicDeleteUTest::setUp(void)
{
}

void BasicDeleteUTest::tearDown(void)
{
}

// 9 more from CogServerNode and gorp; 2 for *-TruthValueKey-* and
// *-IsKeyFlag-*; 1 more for *-open-*
int basic_offset = 9 + 3;

// ============================================================

void BasicDeleteUTest::test_duplicates(void)
{
	logger().debug("BEGIN TEST: %s", __FUNCTION__);

	AtomSpacePtr as = createAtomSpace();

	Handle hsn = as->add_node(COG_STORAGE_NODE, std::string(uri));
	StorageNodePtr store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// Clear out left-over junk, just in case.
	::kill_data(store.get(), _test_asp.get());

	Handle ha(as->add_node(CONCEPT_NODE, "a"));
	Handle hb(as->add_node(CONCEPT_NODE, "b"));
	Handle li(as->add_link(LIST_LINK, ha, hb, ha, hb));

	// Store the link.
	store->store_atom(li);
	store->barrier();

	// Verify the atomspace size (+1 for *-open-* predicate)
	TS_ASSERT_EQUALS(as->get_size(), 5);

	// Remove the link. Since the link has duplicate atoms
	// in it, this can cause trouble, depending on how the
	// incoming set is managed.
	store->remove_atom(as, li);
	TS_ASSERT_EQUALS(as->get_size(), 5);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(li);
	TS_ASSERT_EQUALS(as->get_size(), 5);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-close-* predicate from above
	TS_ASSERT_EQUALS(as->get_size(), 4 + basic_offset);

	::kill_data(store.get(), _test_asp.get());
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());

	logger().debug("END TEST: %s", __FUNCTION__);
}

// ============================================================

void BasicDeleteUTest::test_recursive(void)
{
	logger().debug("BEGIN TEST: %s", __FUNCTION__);

	AtomSpacePtr as = createAtomSpace();

	Handle hsn = as->add_node(COG_STORAGE_NODE, std::string(uri));
	StorageNodePtr store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// Clear out left-over junk, just in case.
	::kill_data(store.get(), _test_asp.get());

	Handle ha(as->add_node(CONCEPT_NODE, "a"));
	Handle hb(as->add_node(CONCEPT_NODE, "b"));
	Handle li(as->add_link(LIST_LINK, ha, hb));
	Handle lj(as->add_link(LIST_LINK, ha, li));

	// Store the link.
	store->store_atom(lj);
	store->barrier();

	printf("Atomspace size=%lu\n", as->get_size());
	HandleSeq hset;
	as->get_root_set_by_type(hset, ATOM, true);
	for (const Handle& h : hset)
		printf("Atomspace contains: %s\n", h->to_short_string().c_str());

	// Verify the atomspace size (+1 for *-open-* predicate)
	TS_ASSERT_EQUALS(as->get_size(), 6);

	// Remove (Concept "a"). Since this appears twice, recursion
	// will hit it twice, in stange ways, and maybe crash.
	store->remove_atom(as, ha, true);

	printf("Post-delete atomspace size=%lu\n", as->get_size());
	hset.clear();
	as->get_root_set_by_type(hset, ATOM, true);
	for (const Handle& h : hset)
		printf("Post delete: %s\n", h->to_short_string().c_str());

	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-close-* predicate from above
	TS_ASSERT_EQUALS(as->get_size(), 3 + basic_offset);

	// --------------------------------------
	// Do it again, but with a different order.
	ha = as->add_node(CONCEPT_NODE, "a");
	li = as->add_link(LIST_LINK, ha, hb);
	lj = as->add_link(LIST_LINK, li, ha);

	store->store_atom(lj);
	store->barrier();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-close-* predicate from above
	TS_ASSERT_EQUALS(as->get_size(), 6 + basic_offset);

	store->remove_atom(as, ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-close-* predicate from above
	TS_ASSERT_EQUALS(as->get_size(), 3 + basic_offset);

	// --------------------------------------
	// Do it again, but with a complex order.
	ha = as->add_node(CONCEPT_NODE, "a");
	li = as->add_link(LIST_LINK, ha, hb);
	lj = as->add_link(LIST_LINK, ha, li, ha);

	store->store_atom(lj);
	store->barrier();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-close-* predicate from above
	TS_ASSERT_EQUALS(as->get_size(), 6 + basic_offset);

	store->remove_atom(as, ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-close-* predicate from above
	TS_ASSERT_EQUALS(as->get_size(), 3 + basic_offset);

	// --------------------------------------
	// Do it again, but more deeply.
	ha = as->add_node(CONCEPT_NODE, "a");
	li = as->add_link(LIST_LINK, ha, hb);
	lj = as->add_link(LIST_LINK, ha, li, ha);
	Handle lk = as->add_link(LIST_LINK, ha, lj, ha);

	store->store_atom(lk);
	store->barrier();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-close-* predicate from above
	TS_ASSERT_EQUALS(as->get_size(), 7 + basic_offset);

	store->remove_atom(as, ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-close-* predicate from above
	TS_ASSERT_EQUALS(as->get_size(), 3 + basic_offset);

	// --------------------------------------
	::kill_data(store.get(), _test_asp.get());
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());

	logger().debug("END TEST: %s", __FUNCTION__);
}

// ============================================================

void BasicDeleteUTest::test_key_deletion(void)
{
	logger().debug("BEGIN TEST: %s", __FUNCTION__);

	// Create an Atom with some keys on it.
	AtomSpacePtr as = createAtomSpace();

	Handle hsn = as->add_node(COG_STORAGE_NODE, std::string(uri));
	StorageNodePtr store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// Clear out left-over junk, just in case.
	::kill_data(store.get(), _test_asp.get());

	Handle ha(as->add_node(CONCEPT_NODE, "a"));
	Handle ka(as->add_node(PREDICATE_NODE, "key-a"));
	Handle kb(as->add_node(PREDICATE_NODE, "key-b"));
	Handle kc(as->add_node(PREDICATE_NODE, "key-c"));

	ha = as->set_value(ha, ka, createFloatValue(std::vector<double>({1, 2, 3})));
	ha = as->set_value(ha, kb, createFloatValue(std::vector<double>({4, 5, 6})));
	ha = as->set_value(ha, kc, createFloatValue(std::vector<double>({7, 8, 9})));

	// Store all three keys.
	store->store_atom(ha);
	store->barrier();

	// Verify the atomspace size (+1 for *-IsKeyFlag-*, +1 for *-open-*)
	TS_ASSERT_EQUALS(as->get_size(), 7);

	// Verify that there are three keys
	HandleSet hs = ha->getKeys();
	for (const Handle& k : hs)
		printf("key=%s\n", k->get_name().c_str());
	TS_ASSERT_EQUALS(hs.size(), 3);

	// Attempt to remove one of the keys. Neither storage, nor
	// AtomSpace will allow this.
	store->remove_atom(as, kb);
	as->extract_atom(kb);
	// +1 for *-IsKeyFlag-*, +1 for *-open-*
	TS_ASSERT_EQUALS(as->get_size(), 7);

	// Verify the key was deleted by loading only PredicateNodes.
	as = createAtomSpace();
	hsn = as->add_node(COG_STORAGE_NODE, std::string(uri));
	store = StorageNodeCast(hsn);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->fetch_all_atoms_of_type(PREDICATE_NODE);
	store->barrier();

	// All keys should have been restored.
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-b"));
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-a"));
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-c"));

	// FYI, we should call store->close() at this point, but we
	// don't, because ... most users forget to do this too.
	// So this is a softball check that everything worked, even
	// without the close().

	// -------------------------------------------------------
	// Start again, with a fresh connection.
	// Verify that the delete key is really deleted.

	as = createAtomSpace();

	hsn = as->add_node(COG_STORAGE_NODE, std::string(uri));
	store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// We expect 5 atoms
	printf("Loading the atomspace ...\n");
	store->load_atomspace();
	TS_ASSERT_EQUALS(as->get_size(), 5 + basic_offset);

	// Verify that there are only two keys .. haha no, three.
	// That is because the deleted key gets restored when the
	// values on (Concept "a") are fetched.
	// Try this:
	// (cog-set-values! (ConceptNode "a")(list (cons (PredicateNode "key-b")(FloatValue 4 5 6))(cons (PredicateNode "key-c")(FloatValue 7 8 9))(cons (PredicateNode "key-a")(FloatValue 1 2 3))))
	// (cog-extract! (PredicateNode "key-b"))
	// (cog-get-atoms 'Node #t)
	// ; Excellent! key-b is gone!
	// (cog-keys->alist (ConceptNode "a"))
	// ; Oh hey, key-b is back!
	//
	ha = as->add_node(CONCEPT_NODE, "a");
	hs = ha->getKeys();
	for (const Handle& k : hs)
		printf("key=%s\n", k->get_name().c_str());
	TS_ASSERT_EQUALS(hs.size(), 3);

	::kill_data(store.get(), _test_asp.get());
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());

	logger().debug("END TEST: %s", __FUNCTION__);
}

// ============================================================

// A few more that cannot be deleted.
int full_offset = basic_offset + 3;

// This tests for a nasty off-by-one bug that showed up
// in RocksDB. Make sure it never happens again.
void BasicDeleteUTest::test_key_corruption(void)
{
	logger().debug("BEGIN TEST: %s", __FUNCTION__);

	// Create an Atom with some keys on it.
	AtomSpacePtr as = createAtomSpace();

	Handle hsn = as->add_node(COG_STORAGE_NODE, std::string(uri));
	StorageNodePtr store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// Clear out left-over junk, just in case.
	::kill_data(store.get(), _test_asp.get());

	Handle hb(as->add_node(CONCEPT_NODE, "b"));
	Handle hx(as->add_node(CONCEPT_NODE, "x"));
	Handle kx(as->add_node(PREDICATE_NODE, "key-x"));

	hb = as->set_value(hb, kx, hx);

	// Store everything.
	store->store_atom(hb);
	store->barrier();

	TS_ASSERT_EQUALS(as->get_size(), 6);

	// Remove the value
	store->remove_atom(as, hx);
	as->extract_atom(hx);
	TS_ASSERT_EQUALS(as->get_size(), 5);

	// FYI, we should call store->close() at this point, but we
	// don't, because ... most users forget to do this too.
	// So this is a softball check that everything worked, even
	// without the close().

	// -------------------------------------------------------
	// Start again, with a fresh connection.
	// Verify that the delete key is really deleted.

	as = createAtomSpace();

	hsn = as->add_node(COG_STORAGE_NODE, std::string(uri));
	store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// We expect 5 atoms: b, x, key-x, *-IsKeyFlag-*, StorageNode
	// That's because the (Concept "x") sitting in the Value
	// was never actually deleted; thus it is restored.
	printf("Loading the atomspace ...\n");
	store->load_atomspace();

	HandleSeq hset;
	as->get_root_set_by_type(hset, ATOM, true);
	for (const Handle& h: hset)
		printf("Atomspace contains h=%s\n", h->to_short_string().c_str());

	// Size is 4 because (Concept "x") cannot be deleted.
	TS_ASSERT_EQUALS(as->get_size(), 4 + full_offset);

	hb = as->add_node(CONCEPT_NODE, "b");
	hx = as->add_node(CONCEPT_NODE, "x");
	Handle hy(as->add_node(CONCEPT_NODE, "y"));
	Handle kl(as->add_node(PREDICATE_NODE, "key-li"));

	hb = as->set_value(hb, kl, createLinkValue(ValueSeq({hx, hy})));

	// Store the new value
	store->store_atom(hb);
	store->barrier();

	TS_ASSERT_EQUALS(as->get_size(), 6 + full_offset);

	// Remove the value .. again
	store->remove_atom(as, hx);
	as->extract_atom(hx);
	TS_ASSERT_EQUALS(as->get_size(), 5 + full_offset);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// Size is 6 because (Concept "x") cannot be deleted. +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 7 + full_offset);

	// We are done.
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());

	// -------------------------------------------------------
	// Start again, with a fresh connection.
	// Verify that the delete key is really deleted.

	as = createAtomSpace();

	hsn = as->add_node(COG_STORAGE_NODE, std::string(uri));
	store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// We expect 6 atoms
	printf("Loading the atomspace ...\n");
	store->load_atomspace();
	TS_ASSERT_EQUALS(as->get_size(), 6 + full_offset);

	// Verify that there are two keys
	hb = as->add_node(CONCEPT_NODE, "b");
	HandleSet hs = hb->getKeys();
	for (const Handle& k : hs)
	{
		ValuePtr vp = hb->getValue(k);
		printf("key=%s value=%s\n", k->to_short_string().c_str(),
			vp->to_short_string().c_str());
	}
	TS_ASSERT_EQUALS(hs.size(), 2);

	::kill_data(store.get(), _test_asp.get());
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());

	logger().debug("END TEST: %s", __FUNCTION__);
}

/* ============================= END OF FILE ================= */