From bf87d931362457d27f8cb64f310859fa395a84f4 Mon Sep 17 00:00:00 2001 From: jmjatlanta Date: Mon, 4 Sep 2017 13:33:56 -0500 Subject: [PATCH] Updating cid codes --- blocks/block.c | 2 +- cid/cid.c | 4 +-- include/ipfs/cid/cid.h | 28 +++++++++++----- include/ipfs/repo/fsrepo/journalstore.h | 15 ++++++--- journal/journal.c | 4 +-- repo/fsrepo/fs_repo.c | 2 +- repo/fsrepo/lmdb_datastore.c | 10 +++++- repo/fsrepo/lmdb_journalstore.c | 44 +++++++++++++++---------- test/cid/test_cid.h | 10 +++--- test/exchange/test_bitswap.h | 6 ++-- test/storage/test_blocks.h | 2 +- test/storage/test_datastore.h | 4 +-- 12 files changed, 82 insertions(+), 49 deletions(-) diff --git a/blocks/block.c b/blocks/block.c index e839a7a..e3e2499 100644 --- a/blocks/block.c +++ b/blocks/block.c @@ -134,7 +134,7 @@ int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, stru return 0; } - block->cid = ipfs_cid_new(0, hash, 32, CID_PROTOBUF); + block->cid = ipfs_cid_new(0, hash, 32, CID_DAG_PROTOBUF); if (block->cid == NULL) { return 0; } diff --git a/cid/cid.c b/cid/cid.c index cc2f3e4..089d1d1 100644 --- a/cid/cid.c +++ b/cid/cid.c @@ -174,7 +174,7 @@ int ipfs_cid_decode_hash_from_base58(const unsigned char* incoming, size_t incom if (retVal == 0) return 0; // now we have the hash, build the object - *cid = ipfs_cid_new(0, &hash[2], hash_length - 2, CID_PROTOBUF); + *cid = ipfs_cid_new(0, &hash[2], hash_length - 2, CID_DAG_PROTOBUF); return *cid != NULL; } @@ -255,7 +255,7 @@ int ipfs_cid_cast(const unsigned char* incoming, size_t incoming_size, struct Ci if (incoming_size == 34 && incoming[0] == 18 && incoming[1] == 32) { // this is a multihash cid->hash_length = mh_multihash_length(incoming, incoming_size); - cid->codec = CID_PROTOBUF; + cid->codec = CID_DAG_PROTOBUF; cid->version = 0; mh_multihash_digest(incoming, incoming_size, &cid->hash, &cid->hash_length); diff --git a/include/ipfs/cid/cid.h b/include/ipfs/cid/cid.h index 503bca9..1990748 100644 --- a/include/ipfs/cid/cid.h +++ b/include/ipfs/cid/cid.h @@ -8,12 +8,22 @@ #include #include "protobuf.h" -#define CID_PROTOBUF 0x70 -#define CID_CBOR 0x71 -#define CID_RAW 0x72 -#define CID_JSON 0x73 +// these are multicodec packed content types. They should match +// the codes described in the authoratative document: +// https://github.com/multiformats/multicodec/blob/master/table.csv +#define CID_RAW 0x55 +#define CID_DAG_PROTOBUF 0x70 +#define CID_DAG_CBOR 0x71 +#define CID_GIT_RAW 0x78 #define CID_ETHEREUM_BLOCK 0x90 -#define CID_ETHEREUM_TX 0x91 +#define CID_ETHEREUM_BLOCKLIST 0x91 +#define CID_ETHEREUM_TRIE 0x92 +#define CID_ETHEREUM_TX 0x93 +#define CID_ETHEREUM_TX_RECEIPT_TRIE 0x94 +#define CID_ETHEREUM_TX_RECEIPT 0x95 +#define CID_ETHEREUM_STATE_TRIE 0x96 +#define CID_ETHEREUM_ACCOUNT_SNAPSHOT 0x97 +#define CID_ETHEREUM_STORAGE_TRIE 0x98 #define CID_BITCOIN_BLOCK 0xb0 #define CID_BITCOIN_TX 0xb1 #define CID_ZCASH_BLOCK 0xc0 @@ -27,10 +37,10 @@ */ struct Cid { - int version; - char codec; - unsigned char* hash; // a multihash - size_t hash_length; + int version; // CID version + int codec; // codec used (i.e. CID_RAW, CID_PROTOBUF + unsigned char* hash; // the multihash + size_t hash_length; // the length of hash }; struct CidSet { diff --git a/include/ipfs/repo/fsrepo/journalstore.h b/include/ipfs/repo/fsrepo/journalstore.h index ab623c0..aa3bc8f 100644 --- a/include/ipfs/repo/fsrepo/journalstore.h +++ b/include/ipfs/repo/fsrepo/journalstore.h @@ -10,12 +10,17 @@ #include "libp2p/db/datastore.h" struct JournalRecord { - unsigned long long timestamp; - int pin; - uint8_t *hash; - size_t hash_size; + unsigned long long timestamp; // the timestamp of the file + int pin; // true if it is to be stored, false if it is to be deleted + int pending; // true if we do not have this file yet + uint8_t *hash; // the hash of the block (file) + size_t hash_size; // the size of the hash }; +struct JournalRecord* lmdb_journal_record_new(); + +int lmdb_journal_record_free(struct JournalRecord* rec); + /** * Open a cursor to the journalstore table */ @@ -33,4 +38,4 @@ int repo_journalstore_cursor_close(struct Datastore* datastore, void* cursor); int journal_record_free(struct JournalRecord* rec); -int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, unsigned long long timestamp, const uint8_t *hash, size_t hash_size); +int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, struct JournalRecord* record); diff --git a/journal/journal.c b/journal/journal.c index 8b582a7..c7547db 100644 --- a/journal/journal.c +++ b/journal/journal.c @@ -94,7 +94,7 @@ int ipfs_journal_free_records(struct Libp2pVector* records) { if (records != NULL) { for (int i = 0; i < records->total; i++) { struct JournalRecord* rec = (struct JournalRecord*)libp2p_utils_vector_get(records, i); - journal_record_free(rec); + lmdb_journal_record_free(rec); } libp2p_utils_vector_free(records); } @@ -319,7 +319,7 @@ int ipfs_journal_handle_message(const uint8_t* incoming, size_t incoming_size, s case (JOURNAL_ENTRY_NEEDED): { // go get a file struct Block* block = NULL; - struct Cid* cid = ipfs_cid_new(0, curr->hash, curr->hash_size, CID_PROTOBUF); + struct Cid* cid = ipfs_cid_new(0, curr->hash, curr->hash_size, CID_DAG_PROTOBUF); // debugging char* str = NULL; libp2p_logger_debug("journal", "Looking for block %s.\n", ipfs_cid_to_string(cid, &str)); diff --git a/repo/fsrepo/fs_repo.c b/repo/fsrepo/fs_repo.c index cf1e570..0abf04d 100644 --- a/repo/fsrepo/fs_repo.c +++ b/repo/fsrepo/fs_repo.c @@ -818,7 +818,7 @@ int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, s libp2p_datastore_record_free(datastore_record); // now get the block from the blockstore - struct Cid* cid = ipfs_cid_new(0, hash, hash_length, CID_PROTOBUF); + struct Cid* cid = ipfs_cid_new(0, hash, hash_length, CID_DAG_PROTOBUF); if (cid == NULL) return 0; struct Blockstore* blockstore = ipfs_blockstore_new(fs_repo); diff --git a/repo/fsrepo/lmdb_datastore.c b/repo/fsrepo/lmdb_datastore.c index 0015b5e..21af022 100644 --- a/repo/fsrepo/lmdb_datastore.c +++ b/repo/fsrepo/lmdb_datastore.c @@ -181,7 +181,15 @@ int repo_fsrepo_lmdb_put(unsigned const char* key, size_t key_size, unsigned cha retVal = mdb_put(mdb_txn, mdb_dbi, &db_key, &db_value, MDB_NODUPDATA | MDB_NOOVERWRITE); if (retVal == 0) { // the normal case - lmdb_journalstore_journal_add(mdb_txn, timestamp, key, key_size); + // add it to the journalstore + struct JournalRecord* rec = lmdb_journal_record_new(); + rec->hash = (uint8_t*)key; + rec->hash_size = key_size; + rec->timestamp = timestamp; + rec->pending = 1; + rec->pin = 1; + lmdb_journalstore_journal_add(mdb_txn, rec); + lmdb_journal_record_free(rec); retVal = 1; } else { if (retVal == MDB_KEYEXIST) // We tried to add a key that already exists. Skip. diff --git a/repo/fsrepo/lmdb_journalstore.c b/repo/fsrepo/lmdb_journalstore.c index ffaecf2..a9844e7 100644 --- a/repo/fsrepo/lmdb_journalstore.c +++ b/repo/fsrepo/lmdb_journalstore.c @@ -7,8 +7,19 @@ #include "ipfs/repo/fsrepo/journalstore.h" #include "ipfs/repo/fsrepo/lmdb_datastore.h" +struct JournalRecord* lmdb_journal_record_new() { + struct JournalRecord* rec = (struct JournalRecord*) malloc(sizeof(struct JournalRecord)); + if (rec != NULL) { + rec->hash = NULL; + rec->hash_size = 0; + rec->pending = 0; + rec->pin = 0; + rec->timestamp = 0; + } + return rec; +} -int journal_record_free(struct JournalRecord* rec) { +int lmdb_journal_record_free(struct JournalRecord* rec) { if (rec != NULL) { if (rec->hash != NULL) free(rec->hash); @@ -26,30 +37,26 @@ int journal_record_free(struct JournalRecord* rec) { * @param hash_size the size of the hash * @returns true(1) on success, false(0) otherwise */ -int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, unsigned long long timestamp, const uint8_t *hash, size_t hash_size) { +int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, struct JournalRecord* journal_record) { MDB_dbi mdb_dbi; struct MDB_val db_key; struct MDB_val db_value; - // build the record, which is a timestamp as a key, a byte that is the pin flag, and the hash as the value + // build the record, which is a timestamp as a key uint8_t time_varint[8]; size_t time_varint_size = 0; - varint_encode(timestamp, &time_varint[0], 8, &time_varint_size); + varint_encode(journal_record->timestamp, &time_varint[0], 8, &time_varint_size); - size_t record_size = hash_size + 1; + size_t record_size = journal_record->hash_size + 2; uint8_t record[record_size]; - record[0] = 1; - memcpy(&record[1], hash, hash_size); - - // debug - size_t b58size = 100; - uint8_t *b58key = (uint8_t *) malloc(b58size); - libp2p_crypto_encoding_base58_encode(hash, hash_size, &b58key, &b58size); - libp2p_logger_debug("lmdb_journalstore", "Adding hash %s to journalstore.\n", b58key); - free(b58key); + // Field 1: pin flag + record[0] = journal_record->pin; + // Field 2: pending flag + record[1] = journal_record->pending; + // field 3: hash + memcpy(&record[2], journal_record->hash, journal_record->hash_size); // open the journal table - if (mdb_dbi_open(mdb_txn, "JOURNALSTORE", MDB_DUPSORT | MDB_CREATE, &mdb_dbi) != 0) { return 0; } @@ -138,10 +145,13 @@ int repo_journalstore_cursor_get(struct Datastore* datastore, void* crsr, enum D rec->timestamp = varint_decode(mdb_key.mv_data, mdb_key.mv_size, &varint_size); // pin flag rec->pin = ((uint8_t*)mdb_value.mv_data)[0]; - rec->hash_size = mdb_value.mv_size - 1; + // pending flag + rec->pending = ((uint8_t*)mdb_value.mv_data)[1]; + // hash + rec->hash_size = mdb_value.mv_size - 2; rec->hash = malloc(rec->hash_size); uint8_t *val = (uint8_t*)mdb_value.mv_data; - memcpy(rec->hash, &val[1], rec->hash_size); + memcpy(rec->hash, &val[2], rec->hash_size); return 1; } return 0; diff --git a/test/cid/test_cid.h b/test/cid/test_cid.h index ab34847..0166007 100644 --- a/test/cid/test_cid.h +++ b/test/cid/test_cid.h @@ -12,7 +12,7 @@ int test_cid_new_free() { unsigned char* hash = (unsigned char*)"ABC123"; - struct Cid* cid = ipfs_cid_new(0, hash, strlen((char*)hash), CID_PROTOBUF); + struct Cid* cid = ipfs_cid_new(0, hash, strlen((char*)hash), CID_DAG_PROTOBUF); if (cid == NULL) return 0; @@ -20,7 +20,7 @@ int test_cid_new_free() { if (cid->version != 0) return 0; - if (cid->codec != CID_PROTOBUF) + if (cid->codec != CID_DAG_PROTOBUF) return 0; if (cid->hash_length != strlen((char*)hash)) @@ -62,7 +62,7 @@ int test_cid_cast_multihash() { return 0; if (cid.hash_length != 32) return 0; - if (cid.codec != CID_PROTOBUF) + if (cid.codec != CID_DAG_PROTOBUF) return 0; if (strncmp((char*)hashed, (char*)cid.hash, 32) != 0) return 0; @@ -85,7 +85,7 @@ int test_cid_cast_non_multihash() { // first the version array[0] = 0; // then the codec - array[1] = CID_PROTOBUF; + array[1] = CID_DAG_PROTOBUF; // then the hash memcpy(&array[2], hashed, 32); @@ -99,7 +99,7 @@ int test_cid_cast_non_multihash() { return 0; if (cid.hash_length != 32) return 0; - if (cid.codec != CID_PROTOBUF) + if (cid.codec != CID_DAG_PROTOBUF) return 0; if (strncmp((char*)hashed, (char*)cid.hash, 32) != 0) return 0; diff --git a/test/exchange/test_bitswap.h b/test/exchange/test_bitswap.h index 5e6fcac..0428b45 100644 --- a/test/exchange/test_bitswap.h +++ b/test/exchange/test_bitswap.h @@ -136,7 +136,7 @@ int test_bitswap_retrieve_file() ipfs_import_file(NULL, "/home/parallels/ipfstest/hello_world.txt", &node, localNode, &bytes_written, 0); // build the Cid from the node information - cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF); + cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF); if (cid == NULL) goto exit; @@ -220,7 +220,7 @@ int test_bitswap_retrieve_file_remote() { ipfs_node2->routing->Bootstrap(ipfs_node2->routing); // this does the heavy lifting... - cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF); + cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF); if (!ipfs_node2->exchange->GetBlock(ipfs_node2->exchange, cid, &result)) { libp2p_logger_error("test_bitswap", "GetBlock returned false\n"); goto exit; @@ -445,7 +445,7 @@ int test_bitswap_retrieve_file_third_party() { ipfs_node3->routing->Bootstrap(ipfs_node3->routing); // this does the heavy lifting... - cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF); + cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF); if (!ipfs_node3->exchange->GetBlock(ipfs_node3->exchange, cid, &result)) { libp2p_logger_error("test_bitswap", "GetBlock returned false\n"); goto exit; diff --git a/test/storage/test_blocks.h b/test/storage/test_blocks.h index af5e9cf..cd875ff 100644 --- a/test/storage/test_blocks.h +++ b/test/storage/test_blocks.h @@ -24,7 +24,7 @@ int test_blocks_new() { return 0; } - if (block->cid->codec != CID_PROTOBUF) { + if (block->cid->codec != CID_DAG_PROTOBUF) { ipfs_block_free(block); return 0; } diff --git a/test/storage/test_datastore.h b/test/storage/test_datastore.h index 85fe25f..2307096 100644 --- a/test/storage/test_datastore.h +++ b/test/storage/test_datastore.h @@ -94,14 +94,14 @@ int test_datastore_list_journal() { enum DatastoreCursorOp op = CURSOR_FIRST; do { if (repo_journalstore_cursor_get(fs_repo->config->datastore, crsr, op, &record) == 0) { - journal_record_free(record); + lmdb_journal_record_free(record); record = NULL; } // display record libp2p_logger_debug("test_datastore", "Timestamp: %llu.\n", record->timestamp); libp2p_logger_debug("test_datastore", "Pin: %s.\n", record->pin == 1 ? "Y" : "N"); // free record - journal_record_free(record); + lmdb_journal_record_free(record); record = NULL; op = CURSOR_NEXT; } while (record != NULL);