agorise/c-ipfs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Updating cid codes

Browse source
This commit is contained in:
jmjatlanta 2017-09-04 13:33:56 -05:00
parent 7dbb6fca29
commit bf87d93136
12 changed files with 82 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
blocks/block.c
View file

@ -134,7 +134,7 @@ int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, stru
return 0; return 0;
} }
block->cid = ipfs_cid_new(0, hash, 32, CID_PROTOBUF); block->cid = ipfs_cid_new(0, hash, 32, CID_DAG_PROTOBUF);
if (block->cid == NULL) { if (block->cid == NULL) {
return 0; return 0;
} }

4
cid/cid.c
View file

@ -174,7 +174,7 @@ int ipfs_cid_decode_hash_from_base58(const unsigned char* incoming, size_t incom
if (retVal == 0) if (retVal == 0)
return 0; return 0;
// now we have the hash, build the object // now we have the hash, build the object
*cid = ipfs_cid_new(0, &hash[2], hash_length - 2, CID_PROTOBUF); *cid = ipfs_cid_new(0, &hash[2], hash_length - 2, CID_DAG_PROTOBUF);
return *cid != NULL; return *cid != NULL;
} }
@ -255,7 +255,7 @@ int ipfs_cid_cast(const unsigned char* incoming, size_t incoming_size, struct Ci
if (incoming_size == 34 && incoming[0] == 18 && incoming[1] == 32) { if (incoming_size == 34 && incoming[0] == 18 && incoming[1] == 32) {
// this is a multihash // this is a multihash
cid->hash_length = mh_multihash_length(incoming, incoming_size); cid->hash_length = mh_multihash_length(incoming, incoming_size);
cid->codec = CID_PROTOBUF; cid->codec = CID_DAG_PROTOBUF;
cid->version = 0; cid->version = 0;
mh_multihash_digest(incoming, incoming_size, &cid->hash, &cid->hash_length); mh_multihash_digest(incoming, incoming_size, &cid->hash, &cid->hash_length);

28
include/ipfs/cid/cid.h
View file

@ -8,12 +8,22 @@
#include <stddef.h> #include <stddef.h>
#include "protobuf.h" #include "protobuf.h"
#define CID_PROTOBUF 0x70 // these are multicodec packed content types. They should match
#define CID_CBOR 0x71 // the codes described in the authoratative document:
#define CID_RAW 0x72 // https://github.com/multiformats/multicodec/blob/master/table.csv
#define CID_JSON 0x73 #define CID_RAW 0x55
#define CID_DAG_PROTOBUF 0x70
#define CID_DAG_CBOR 0x71
#define CID_GIT_RAW 0x78
#define CID_ETHEREUM_BLOCK 0x90 #define CID_ETHEREUM_BLOCK 0x90
#define CID_ETHEREUM_TX 0x91 #define CID_ETHEREUM_BLOCKLIST 0x91
#define CID_ETHEREUM_TRIE 0x92
#define CID_ETHEREUM_TX 0x93
#define CID_ETHEREUM_TX_RECEIPT_TRIE 0x94
#define CID_ETHEREUM_TX_RECEIPT 0x95
#define CID_ETHEREUM_STATE_TRIE 0x96
#define CID_ETHEREUM_ACCOUNT_SNAPSHOT 0x97
#define CID_ETHEREUM_STORAGE_TRIE 0x98
#define CID_BITCOIN_BLOCK 0xb0 #define CID_BITCOIN_BLOCK 0xb0
#define CID_BITCOIN_TX 0xb1 #define CID_BITCOIN_TX 0xb1
#define CID_ZCASH_BLOCK 0xc0 #define CID_ZCASH_BLOCK 0xc0
@ -27,10 +37,10 @@
*/ */
struct Cid { struct Cid {
int version; int version; // CID version
char codec; int codec; // codec used (i.e. CID_RAW, CID_PROTOBUF
unsigned char* hash; // a multihash unsigned char* hash; // the multihash
size_t hash_length; size_t hash_length; // the length of hash
}; };
struct CidSet { struct CidSet {

15
include/ipfs/repo/fsrepo/journalstore.h
View file

@ -10,12 +10,17 @@
#include "libp2p/db/datastore.h" #include "libp2p/db/datastore.h"
struct JournalRecord { struct JournalRecord {
unsigned long long timestamp; unsigned long long timestamp; // the timestamp of the file
int pin; int pin; // true if it is to be stored, false if it is to be deleted
uint8_t *hash; int pending; // true if we do not have this file yet
size_t hash_size; uint8_t *hash; // the hash of the block (file)
size_t hash_size; // the size of the hash
}; };
struct JournalRecord* lmdb_journal_record_new();
int lmdb_journal_record_free(struct JournalRecord* rec);
/** /**
* Open a cursor to the journalstore table * Open a cursor to the journalstore table
*/ */
@ -33,4 +38,4 @@ int repo_journalstore_cursor_close(struct Datastore* datastore, void* cursor);
int journal_record_free(struct JournalRecord* rec); int journal_record_free(struct JournalRecord* rec);
int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, unsigned long long timestamp, const uint8_t *hash, size_t hash_size); int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, struct JournalRecord* record);

4
journal/journal.c
View file

@ -94,7 +94,7 @@ int ipfs_journal_free_records(struct Libp2pVector* records) {
if (records != NULL) { if (records != NULL) {
for (int i = 0; i < records->total; i++) { for (int i = 0; i < records->total; i++) {
struct JournalRecord* rec = (struct JournalRecord*)libp2p_utils_vector_get(records, i); struct JournalRecord* rec = (struct JournalRecord*)libp2p_utils_vector_get(records, i);
journal_record_free(rec); lmdb_journal_record_free(rec);
} }
libp2p_utils_vector_free(records); libp2p_utils_vector_free(records);
} }
@ -319,7 +319,7 @@ int ipfs_journal_handle_message(const uint8_t* incoming, size_t incoming_size, s
case (JOURNAL_ENTRY_NEEDED): { case (JOURNAL_ENTRY_NEEDED): {
// go get a file // go get a file
struct Block* block = NULL; struct Block* block = NULL;
struct Cid* cid = ipfs_cid_new(0, curr->hash, curr->hash_size, CID_PROTOBUF); struct Cid* cid = ipfs_cid_new(0, curr->hash, curr->hash_size, CID_DAG_PROTOBUF);
// debugging // debugging
char* str = NULL; char* str = NULL;
libp2p_logger_debug("journal", "Looking for block %s.\n", ipfs_cid_to_string(cid, &str)); libp2p_logger_debug("journal", "Looking for block %s.\n", ipfs_cid_to_string(cid, &str));

2
repo/fsrepo/fs_repo.c
View file

@ -818,7 +818,7 @@ int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, s
libp2p_datastore_record_free(datastore_record); libp2p_datastore_record_free(datastore_record);
// now get the block from the blockstore // now get the block from the blockstore
struct Cid* cid = ipfs_cid_new(0, hash, hash_length, CID_PROTOBUF); struct Cid* cid = ipfs_cid_new(0, hash, hash_length, CID_DAG_PROTOBUF);
if (cid == NULL) if (cid == NULL)
return 0; return 0;
struct Blockstore* blockstore = ipfs_blockstore_new(fs_repo); struct Blockstore* blockstore = ipfs_blockstore_new(fs_repo);

10
repo/fsrepo/lmdb_datastore.c
View file

@ -181,7 +181,15 @@ int repo_fsrepo_lmdb_put(unsigned const char* key, size_t key_size, unsigned cha
retVal = mdb_put(mdb_txn, mdb_dbi, &db_key, &db_value, MDB_NODUPDATA | MDB_NOOVERWRITE); retVal = mdb_put(mdb_txn, mdb_dbi, &db_key, &db_value, MDB_NODUPDATA | MDB_NOOVERWRITE);
if (retVal == 0) { if (retVal == 0) {
// the normal case // the normal case
lmdb_journalstore_journal_add(mdb_txn, timestamp, key, key_size); // add it to the journalstore
struct JournalRecord* rec = lmdb_journal_record_new();
rec->hash = (uint8_t*)key;
rec->hash_size = key_size;
rec->timestamp = timestamp;
rec->pending = 1;
rec->pin = 1;
lmdb_journalstore_journal_add(mdb_txn, rec);
lmdb_journal_record_free(rec);
retVal = 1; retVal = 1;
} else { } else {
if (retVal == MDB_KEYEXIST) // We tried to add a key that already exists. Skip. if (retVal == MDB_KEYEXIST) // We tried to add a key that already exists. Skip.

44
repo/fsrepo/lmdb_journalstore.c
View file

@ -7,8 +7,19 @@
#include "ipfs/repo/fsrepo/journalstore.h" #include "ipfs/repo/fsrepo/journalstore.h"
#include "ipfs/repo/fsrepo/lmdb_datastore.h" #include "ipfs/repo/fsrepo/lmdb_datastore.h"
struct JournalRecord* lmdb_journal_record_new() {
struct JournalRecord* rec = (struct JournalRecord*) malloc(sizeof(struct JournalRecord));
if (rec != NULL) {
rec->hash = NULL;
rec->hash_size = 0;
rec->pending = 0;
rec->pin = 0;
rec->timestamp = 0;
}
return rec;
}
int journal_record_free(struct JournalRecord* rec) { int lmdb_journal_record_free(struct JournalRecord* rec) {
if (rec != NULL) { if (rec != NULL) {
if (rec->hash != NULL) if (rec->hash != NULL)
free(rec->hash); free(rec->hash);
@ -26,30 +37,26 @@ int journal_record_free(struct JournalRecord* rec) {
* @param hash_size the size of the hash * @param hash_size the size of the hash
* @returns true(1) on success, false(0) otherwise * @returns true(1) on success, false(0) otherwise
*/ */
int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, unsigned long long timestamp, const uint8_t *hash, size_t hash_size) { int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, struct JournalRecord* journal_record) {
MDB_dbi mdb_dbi; MDB_dbi mdb_dbi;
struct MDB_val db_key; struct MDB_val db_key;
struct MDB_val db_value; struct MDB_val db_value;
// build the record, which is a timestamp as a key, a byte that is the pin flag, and the hash as the value // build the record, which is a timestamp as a key
uint8_t time_varint[8]; uint8_t time_varint[8];
size_t time_varint_size = 0; size_t time_varint_size = 0;
varint_encode(timestamp, &time_varint[0], 8, &time_varint_size); varint_encode(journal_record->timestamp, &time_varint[0], 8, &time_varint_size);
size_t record_size = hash_size + 1; size_t record_size = journal_record->hash_size + 2;
uint8_t record[record_size]; uint8_t record[record_size];
record[0] = 1; // Field 1: pin flag
memcpy(&record[1], hash, hash_size); record[0] = journal_record->pin;
// Field 2: pending flag
// debug record[1] = journal_record->pending;
size_t b58size = 100; // field 3: hash
uint8_t *b58key = (uint8_t *) malloc(b58size); memcpy(&record[2], journal_record->hash, journal_record->hash_size);
libp2p_crypto_encoding_base58_encode(hash, hash_size, &b58key, &b58size);
libp2p_logger_debug("lmdb_journalstore", "Adding hash %s to journalstore.\n", b58key);
free(b58key);
// open the journal table // open the journal table
if (mdb_dbi_open(mdb_txn, "JOURNALSTORE", MDB_DUPSORT | MDB_CREATE, &mdb_dbi) != 0) { if (mdb_dbi_open(mdb_txn, "JOURNALSTORE", MDB_DUPSORT | MDB_CREATE, &mdb_dbi) != 0) {
return 0; return 0;
} }
@ -138,10 +145,13 @@ int repo_journalstore_cursor_get(struct Datastore* datastore, void* crsr, enum D
rec->timestamp = varint_decode(mdb_key.mv_data, mdb_key.mv_size, &varint_size); rec->timestamp = varint_decode(mdb_key.mv_data, mdb_key.mv_size, &varint_size);
// pin flag // pin flag
rec->pin = ((uint8_t*)mdb_value.mv_data)[0]; rec->pin = ((uint8_t*)mdb_value.mv_data)[0];
rec->hash_size = mdb_value.mv_size - 1; // pending flag
rec->pending = ((uint8_t*)mdb_value.mv_data)[1];
// hash
rec->hash_size = mdb_value.mv_size - 2;
rec->hash = malloc(rec->hash_size); rec->hash = malloc(rec->hash_size);
uint8_t *val = (uint8_t*)mdb_value.mv_data; uint8_t *val = (uint8_t*)mdb_value.mv_data;
memcpy(rec->hash, &val[1], rec->hash_size); memcpy(rec->hash, &val[2], rec->hash_size);
return 1; return 1;
} }
return 0; return 0;

10
test/cid/test_cid.h
View file

@ -12,7 +12,7 @@ int test_cid_new_free() {
unsigned char* hash = (unsigned char*)"ABC123"; unsigned char* hash = (unsigned char*)"ABC123";
struct Cid* cid = ipfs_cid_new(0, hash, strlen((char*)hash), CID_PROTOBUF); struct Cid* cid = ipfs_cid_new(0, hash, strlen((char*)hash), CID_DAG_PROTOBUF);
if (cid == NULL) if (cid == NULL)
return 0; return 0;
@ -20,7 +20,7 @@ int test_cid_new_free() {
if (cid->version != 0) if (cid->version != 0)
return 0; return 0;
if (cid->codec != CID_PROTOBUF) if (cid->codec != CID_DAG_PROTOBUF)
return 0; return 0;
if (cid->hash_length != strlen((char*)hash)) if (cid->hash_length != strlen((char*)hash))
@ -62,7 +62,7 @@ int test_cid_cast_multihash() {
return 0; return 0;
if (cid.hash_length != 32) if (cid.hash_length != 32)
return 0; return 0;
if (cid.codec != CID_PROTOBUF) if (cid.codec != CID_DAG_PROTOBUF)
return 0; return 0;
if (strncmp((char*)hashed, (char*)cid.hash, 32) != 0) if (strncmp((char*)hashed, (char*)cid.hash, 32) != 0)
return 0; return 0;
@ -85,7 +85,7 @@ int test_cid_cast_non_multihash() {
// first the version // first the version
array[0] = 0; array[0] = 0;
// then the codec // then the codec
array[1] = CID_PROTOBUF; array[1] = CID_DAG_PROTOBUF;
// then the hash // then the hash
memcpy(&array[2], hashed, 32); memcpy(&array[2], hashed, 32);
@ -99,7 +99,7 @@ int test_cid_cast_non_multihash() {
return 0; return 0;
if (cid.hash_length != 32) if (cid.hash_length != 32)
return 0; return 0;
if (cid.codec != CID_PROTOBUF) if (cid.codec != CID_DAG_PROTOBUF)
return 0; return 0;
if (strncmp((char*)hashed, (char*)cid.hash, 32) != 0) if (strncmp((char*)hashed, (char*)cid.hash, 32) != 0)
return 0; return 0;

6
test/exchange/test_bitswap.h
View file

@ -136,7 +136,7 @@ int test_bitswap_retrieve_file()
ipfs_import_file(NULL, "/home/parallels/ipfstest/hello_world.txt", &node, localNode, &bytes_written, 0); ipfs_import_file(NULL, "/home/parallels/ipfstest/hello_world.txt", &node, localNode, &bytes_written, 0);
// build the Cid from the node information // build the Cid from the node information
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF); cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF);
if (cid == NULL) if (cid == NULL)
goto exit; goto exit;
@ -220,7 +220,7 @@ int test_bitswap_retrieve_file_remote() {
ipfs_node2->routing->Bootstrap(ipfs_node2->routing); ipfs_node2->routing->Bootstrap(ipfs_node2->routing);
// this does the heavy lifting... // this does the heavy lifting...
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF); cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF);
if (!ipfs_node2->exchange->GetBlock(ipfs_node2->exchange, cid, &result)) { if (!ipfs_node2->exchange->GetBlock(ipfs_node2->exchange, cid, &result)) {
libp2p_logger_error("test_bitswap", "GetBlock returned false\n"); libp2p_logger_error("test_bitswap", "GetBlock returned false\n");
goto exit; goto exit;
@ -445,7 +445,7 @@ int test_bitswap_retrieve_file_third_party() {
ipfs_node3->routing->Bootstrap(ipfs_node3->routing); ipfs_node3->routing->Bootstrap(ipfs_node3->routing);
// this does the heavy lifting... // this does the heavy lifting...
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF); cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF);
if (!ipfs_node3->exchange->GetBlock(ipfs_node3->exchange, cid, &result)) { if (!ipfs_node3->exchange->GetBlock(ipfs_node3->exchange, cid, &result)) {
libp2p_logger_error("test_bitswap", "GetBlock returned false\n"); libp2p_logger_error("test_bitswap", "GetBlock returned false\n");
goto exit; goto exit;

2
test/storage/test_blocks.h
View file

@ -24,7 +24,7 @@ int test_blocks_new() {
return 0; return 0;
} }
if (block->cid->codec != CID_PROTOBUF) { if (block->cid->codec != CID_DAG_PROTOBUF) {
ipfs_block_free(block); ipfs_block_free(block);
return 0; return 0;
} }

4
test/storage/test_datastore.h
View file

@ -94,14 +94,14 @@ int test_datastore_list_journal() {
enum DatastoreCursorOp op = CURSOR_FIRST; enum DatastoreCursorOp op = CURSOR_FIRST;
do { do {
if (repo_journalstore_cursor_get(fs_repo->config->datastore, crsr, op, &record) == 0) { if (repo_journalstore_cursor_get(fs_repo->config->datastore, crsr, op, &record) == 0) {
journal_record_free(record); lmdb_journal_record_free(record);
record = NULL; record = NULL;
} }
// display record // display record
libp2p_logger_debug("test_datastore", "Timestamp: %llu.\n", record->timestamp); libp2p_logger_debug("test_datastore", "Timestamp: %llu.\n", record->timestamp);
libp2p_logger_debug("test_datastore", "Pin: %s.\n", record->pin == 1 ? "Y" : "N"); libp2p_logger_debug("test_datastore", "Pin: %s.\n", record->pin == 1 ? "Y" : "N");
// free record // free record
journal_record_free(record); lmdb_journal_record_free(record);
record = NULL; record = NULL;
op = CURSOR_NEXT; op = CURSOR_NEXT;
} while (record != NULL); } while (record != NULL);