BasicDeleteUTest.cxxtest

/*
 * tests/persist/rocks/BasicDeleteUTest.cxxtest
 *
 * Verify that deletion of keys works, even if some Atoms are still
 * using those keys.
 *
 * Copyright (C) 2008, 2009, 2019, 2021 Linas Vepstas <linasvepstas@gmail.com>
 *
 * LICENSE:
 * SPDX-License-Identifier: AGPL-3.0-or-later
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#include <cstdio>
#include <filesystem>

#include <opencog/atoms/base/Atom.h>
#include <opencog/atoms/base/Link.h>
#include <opencog/atoms/base/Node.h>
#include <opencog/atoms/atom_types/atom_types.h>
#include <opencog/atoms/value/FloatValue.h>
#include <opencog/atoms/value/LinkValue.h>
#include <opencog/atoms/value/VoidValue.h>
#include <opencog/atomspace/AtomSpace.h>

#include <opencog/persist/api/StorageNode.h>
#include <opencog/persist/rocks-types/atom_types.h>

#include <opencog/util/Logger.h>

using namespace opencog;

class BasicDeleteUTest :  public CxxTest::TestSuite
{
	private:
		std::string uri;

	public:

		BasicDeleteUTest(void)
		{
			logger().set_level(Logger::INFO);
			// logger().set_level(Logger::DEBUG);
			logger().set_print_to_stdout_flag(true);

			uri = "rocks:///tmp/cog-rocks-basic-delete-utest";
		}

		~BasicDeleteUTest()
		{
			// erase the log file if no assertions failed
			if (!CxxTest::TestTracker::tracker().suiteFailed())
			{
				std::remove(logger().get_filename().c_str());
				// Also remove the database directory
				// URI is "rocks:///tmp/..." so path starts at position 8
				std::string dbpath = uri.substr(8);
				std::filesystem::remove_all(dbpath);
			}
		}

		void setUp(void);
		void tearDown(void);

		void test_duplicates(void);
		void test_recursive(void);
		void test_key_deletion(void);
		void test_key_corruption(void);
};

/*
 * This is called once before each test, for each test (!!)
 */
void BasicDeleteUTest::setUp(void)
{
}

void BasicDeleteUTest::tearDown(void)
{
}

// ============================================================

void BasicDeleteUTest::test_duplicates(void)
{
	logger().debug("BEGIN TEST: %s", __FUNCTION__);

	AtomSpacePtr as = createAtomSpace();

	Handle hsn = as->add_node(ROCKS_STORAGE_NODE, std::string(uri));
	StorageNodePtr store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// Clear out left-over junk, just in case.
	store->erase();

	Handle ha(as->add_node(CONCEPT_NODE, "a"));
	Handle hb(as->add_node(CONCEPT_NODE, "b"));
	Handle li(as->add_link(LIST_LINK, ha, hb, ha, hb));

	// Store the link.
	store->store_atom(li);
	store->barrier();

	// Verify the atomspace size: 2 concepts + 1 link + 1 storage node + 1 *-open-*
	TS_ASSERT_EQUALS(as->get_size(), 5);

	// Remove the link. Since the link has duplicate atoms
	// in it, this can cause trouble, depending on how the
	// incoming set is managed.
	store->remove_atom(as.get(), li);
	// +1 for *-open-*
	TS_ASSERT_EQUALS(as->get_size(), 5);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(li);
	// +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 5);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 5);

	store->erase();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as = nullptr;

	logger().debug("END TEST: %s", __FUNCTION__);
}

// ============================================================

void BasicDeleteUTest::test_recursive(void)
{
	logger().debug("BEGIN TEST: %s", __FUNCTION__);

	AtomSpacePtr as = createAtomSpace();

	Handle hsn = as->add_node(ROCKS_STORAGE_NODE, std::string(uri));
	StorageNodePtr store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// Clear out left-over junk, just in case.
	store->erase();

	Handle ha(as->add_node(CONCEPT_NODE, "a"));
	Handle hb(as->add_node(CONCEPT_NODE, "b"));
	Handle li(as->add_link(LIST_LINK, ha, hb));
	Handle lj(as->add_link(LIST_LINK, ha, li));

	// Store the link.
	store->store_atom(lj);
	store->barrier();

	printf("Atomspace size=%lu\n", as->get_size());
	HandleSeq hset;
	as->get_root_set_by_type(hset, ATOM, true);
	for (const Handle& h : hset)
		printf("Atomspace contains: %s\n", h->to_short_string().c_str());

	// Verify the atomspace size (+1 for *-open-*)
	TS_ASSERT_EQUALS(as->get_size(), 6);

	// Remove (Concept "a"). Since this appears twice, recursion
	// will hit it twice, in strange ways, and maybe crash.
	store->remove_atom(as.get(), ha, true);

	printf("Post-delete atomspace size=%lu\n", as->get_size());
	hset.clear();
	as->get_root_set_by_type(hset, ATOM, true);
	for (const Handle& h : hset)
		printf("Post delete: %s\n", h->to_short_string().c_str());

	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 4);

	// --------------------------------------
	// Do it again, but with a different order.
	ha = as->add_node(CONCEPT_NODE, "a");
	li = as->add_link(LIST_LINK, ha, hb);
	lj = as->add_link(LIST_LINK, li, ha);

	store->store_atom(lj);
	store->barrier();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 7);

	store->remove_atom(as.get(), ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 4);

	// --------------------------------------
	// Do it again, but with a complex order.
	ha = as->add_node(CONCEPT_NODE, "a");
	li = as->add_link(LIST_LINK, ha, hb);
	lj = as->add_link(LIST_LINK, ha, li, ha);

	store->store_atom(lj);
	store->barrier();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 7);

	store->remove_atom(as.get(), ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 4);

	// --------------------------------------
	// Do it again, but more deeply.
	ha = as->add_node(CONCEPT_NODE, "a");
	li = as->add_link(LIST_LINK, ha, hb);
	lj = as->add_link(LIST_LINK, ha, li, ha);
	Handle lk = as->add_link(LIST_LINK, ha, lj, ha);

	store->store_atom(lk);
	store->barrier();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 8);

	store->remove_atom(as.get(), ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as->extract_atom(ha, true);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 4);

	// --------------------------------------
	store->erase();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as = nullptr;

	logger().debug("END TEST: %s", __FUNCTION__);
}

// ============================================================

void BasicDeleteUTest::test_key_deletion(void)
{
	logger().debug("BEGIN TEST: %s", __FUNCTION__);

	// Create an Atom with some keys on it.
	AtomSpacePtr as = createAtomSpace();

	Handle hsn = as->add_node(ROCKS_STORAGE_NODE, std::string(uri));
	StorageNodePtr store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// Clear out left-over junk, just in case.
	store->erase();

	Handle ha(as->add_node(CONCEPT_NODE, "a"));
	Handle ka(as->add_node(PREDICATE_NODE, "key-a"));
	Handle kb(as->add_node(PREDICATE_NODE, "key-b"));
	Handle kc(as->add_node(PREDICATE_NODE, "key-c"));

	ha = as->set_value(ha, ka, createFloatValue(std::vector<double>({1, 2, 3})));
	ha = as->set_value(ha, kb, createFloatValue(std::vector<double>({4, 5, 6})));
	ha = as->set_value(ha, kc, createFloatValue(std::vector<double>({7, 8, 9})));

	// Store all three keys.
	store->store_atom(ha);
	store->barrier();

	// Verify the atomspace size (+1 for *-IsKeyFlag-*, +1 for *-open-*)
	TS_ASSERT_EQUALS(as->get_size(), 7);

	// Verify that there are three keys
	HandleSet hs = ha->getKeys();
	for (const Handle& k : hs)
		printf("key=%s\n", k->get_name().c_str());
	TS_ASSERT_EQUALS(hs.size(), 3);

	// Attempt to remove one of the keys. Should fail.
	store->remove_atom(as.get(), kb);
	as->extract_atom(kb);
	// +1 for *-IsKeyFlag-*, +1 for *-open-*
	TS_ASSERT_EQUALS(as->get_size(), 7);

	// Verify the key was NOT deleted by loading only PredicateNodes.
	as = nullptr;
	as = createAtomSpace();
	hsn = as->add_node(ROCKS_STORAGE_NODE, std::string(uri));
	store = StorageNodeCast(hsn);
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->fetch_all_atoms_of_type(PREDICATE_NODE);

	// All three keys need to still be here.
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-b"));
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-a"));
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-c"));

	// FYI, we should call store->close() at this point, but we
	// don't, because ... most users forget to do this too.
	// So this is a softball check that everything worked, even
	// without the close().

	// We are done.
	as = nullptr;

	// -------------------------------------------------------
	// Start again, with a fresh connection.
	// Verify that in-use keys cannot be deleted.

	as = createAtomSpace();

	hsn = as->add_node(ROCKS_STORAGE_NODE, std::string(uri));
	store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// We expect 7 atoms (+1 for *-IsKeyFlag-*, +1 for *-open-*)
	printf("Loading the atomspace ...\n");
	store->load_atomspace();
	TS_ASSERT_EQUALS(as->get_size(), 7);
	printf("AtomSpace contents=\n%s\n", as->to_string().c_str());

	// Verify that there are only three keys
	ha = as->add_node(CONCEPT_NODE, "a");
	hs = ha->getKeys();
	for (const Handle& k : hs)
		printf("key=%s\n", k->get_name().c_str());
	TS_ASSERT_EQUALS(hs.size(), 3);

	// All three keys need to still be here.
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-b"));
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-a"));
	TS_ASSERT(nullptr != as->get_node(PREDICATE_NODE, "key-c"));

	store->erase();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as = nullptr;

	logger().debug("END TEST: %s", __FUNCTION__);
}

// ============================================================

// This tests for a nasty off-by-one bug that showed up
// in RocksDB. Make sure it never happens again.
void BasicDeleteUTest::test_key_corruption(void)
{
	logger().debug("BEGIN TEST: %s", __FUNCTION__);

	// Create an Atom with some keys on it.
	AtomSpacePtr as = createAtomSpace();

	Handle hsn = as->add_node(ROCKS_STORAGE_NODE, std::string(uri));
	StorageNodePtr store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// Clear out left-over junk, just in case.
	store->erase();

	Handle hb(as->add_node(CONCEPT_NODE, "b"));
	Handle hx(as->add_node(CONCEPT_NODE, "x"));
	Handle kx(as->add_node(PREDICATE_NODE, "key-x"));

	hb = as->set_value(hb, kx, hx);

	// Store everything.
	store->store_atom(hb);
	store->barrier();

	// Verify the atomspace size (+1 for *-IsKeyFlag-*, +1 for *-open-*)
	TS_ASSERT_EQUALS(as->get_size(), 6);

	// Remove the value
	store->remove_atom(as.get(), hx);
	as->extract_atom(hx);
	// +1 for *-IsKeyFlag-*, +1 for *-open-*
	TS_ASSERT_EQUALS(as->get_size(), 5);

	// FYI, we should call store->close() at this point, but we
	// don't, because ... most users forget to do this too.
	// So this is a softball check that everything worked, even
	// without the close().

	// We are done.
	as = nullptr;

	// -------------------------------------------------------
	// Start again, with a fresh connection.
	// Verify that the delete key is really deleted.

	as = createAtomSpace();

	hsn = as->add_node(ROCKS_STORAGE_NODE, std::string(uri));
	store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// We expect 6 atoms: b, x, key-x, *-IsKeyFlag-*, StorageNode, *-open-*
	// That's because the (Concept "x") sitting in the Value
	// was never actually deleted; thus it is restored.
	printf("Loading the atomspace ...\n");
	store->load_atomspace();

	HandleSeq hset;
	as->get_root_set_by_type(hset, ATOM, true);
	for (const Handle& h: hset)
		printf("Atomspace contains h=%s\n", h->to_short_string().c_str());

	// Size is 6 because (Concept "x") cannot be deleted.
	// +1 for *-IsKeyFlag-*, +1 for *-open-*
	TS_ASSERT_EQUALS(as->get_size(), 6);

	hb = as->add_node(CONCEPT_NODE, "b");
	hx = as->add_node(CONCEPT_NODE, "x");
	Handle hy(as->add_node(CONCEPT_NODE, "y"));
	Handle kl(as->add_node(PREDICATE_NODE, "key-li"));

	hb = as->set_value(hb, kl, createLinkValue(ValueSeq({hx, hy})));

	// Store the new value
	store->store_atom(hb);
	store->barrier();

	// +1 for *-IsKeyFlag-*, +1 for *-open-*
	TS_ASSERT_EQUALS(as->get_size(), 8);

	// Remove the value .. again
	store->remove_atom(as.get(), hx);
	as->extract_atom(hx);
	// +1 for *-IsKeyFlag-*, +1 for *-open-*
	TS_ASSERT_EQUALS(as->get_size(), 7);
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	store->load_atomspace();
	// Size is 9 because (Concept "x") cannot be deleted.
	// +1 for *-IsKeyFlag-*, +1 for *-open-*, +1 for *-close-*
	TS_ASSERT_EQUALS(as->get_size(), 9);

	// We are done.
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as = nullptr;

	// -------------------------------------------------------
	// Start again, with a fresh connection.
	// Verify that the delete key is really deleted.

	as = createAtomSpace();

	hsn = as->add_node(ROCKS_STORAGE_NODE, std::string(uri));
	store = StorageNodeCast(hsn);

	store->setValue(as->add_node(PREDICATE_NODE, "*-open-*"), createVoidValue());
	TS_ASSERT(store->connected())

	// We expect 8 atoms (+1 for *-IsKeyFlag-*, +1 for *-open-*)
	printf("Loading the atomspace ...\n");
	store->load_atomspace();
	TS_ASSERT_EQUALS(as->get_size(), 8);

	// Verify that there are two keys
	hb = as->add_node(CONCEPT_NODE, "b");
	HandleSet hs = hb->getKeys();
	for (const Handle& k : hs)
	{
		ValuePtr vp = hb->getValue(k);
		printf("key=%s value=%s\n", k->to_short_string().c_str(),
			vp->to_short_string().c_str());
	}
	TS_ASSERT_EQUALS(hs.size(), 2);

	store->erase();
	store->setValue(as->add_node(PREDICATE_NODE, "*-close-*"), createVoidValue());
	as = nullptr;

	logger().debug("END TEST: %s", __FUNCTION__);
}

/* ============================= END OF FILE ================= */
opencog-atomspace-rocks / tests / persist / rocks / BasicDeleteUTest.cxxtest
