agorise
/
c-ipfs
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updating cid codes

yamux
jmjatlanta 2017-09-04 13:33:56 -05:00
parent 7dbb6fca29
commit bf87d93136
12 changed files with 82 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
blocks/block.c
View File

@ -134,7 +134,7 @@ int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, stru
return 0;
}
block->cid = ipfs_cid_new(0, hash, 32, CID_PROTOBUF);
block->cid = ipfs_cid_new(0, hash, 32, CID_DAG_PROTOBUF);
if (block->cid == NULL) {
return 0;
}

4
cid/cid.c
View File

@ -174,7 +174,7 @@ int ipfs_cid_decode_hash_from_base58(const unsigned char* incoming, size_t incom
if (retVal == 0)
return 0;
// now we have the hash, build the object
*cid = ipfs_cid_new(0, &hash[2], hash_length - 2, CID_PROTOBUF);
*cid = ipfs_cid_new(0, &hash[2], hash_length - 2, CID_DAG_PROTOBUF);
return *cid != NULL;
}
@ -255,7 +255,7 @@ int ipfs_cid_cast(const unsigned char* incoming, size_t incoming_size, struct Ci
if (incoming_size == 34 && incoming[0] == 18 && incoming[1] == 32) {
// this is a multihash
cid->hash_length = mh_multihash_length(incoming, incoming_size);
cid->codec = CID_PROTOBUF;
cid->codec = CID_DAG_PROTOBUF;
cid->version = 0;
mh_multihash_digest(incoming, incoming_size, &cid->hash, &cid->hash_length);

28
include/ipfs/cid/cid.h
View File

@ -8,12 +8,22 @@
#include <stddef.h>
#include "protobuf.h"
#define CID_PROTOBUF 0x70
#define CID_CBOR 0x71
#define CID_RAW 0x72
#define CID_JSON 0x73
// these are multicodec packed content types. They should match
// the codes described in the authoratative document:
// https://github.com/multiformats/multicodec/blob/master/table.csv
#define CID_RAW 0x55
#define CID_DAG_PROTOBUF 0x70
#define CID_DAG_CBOR 0x71
#define CID_GIT_RAW 0x78
#define CID_ETHEREUM_BLOCK 0x90
#define CID_ETHEREUM_TX 0x91
#define CID_ETHEREUM_BLOCKLIST 0x91
#define CID_ETHEREUM_TRIE 0x92
#define CID_ETHEREUM_TX 0x93
#define CID_ETHEREUM_TX_RECEIPT_TRIE 0x94
#define CID_ETHEREUM_TX_RECEIPT 0x95
#define CID_ETHEREUM_STATE_TRIE 0x96
#define CID_ETHEREUM_ACCOUNT_SNAPSHOT 0x97
#define CID_ETHEREUM_STORAGE_TRIE 0x98
#define CID_BITCOIN_BLOCK 0xb0
#define CID_BITCOIN_TX 0xb1
#define CID_ZCASH_BLOCK 0xc0
@ -27,10 +37,10 @@
*/
struct Cid {
int version;
char codec;
unsigned char* hash; // a multihash
size_t hash_length;
int version; // CID version
int codec; // codec used (i.e. CID_RAW, CID_PROTOBUF
unsigned char* hash; // the multihash
size_t hash_length; // the length of hash
};
struct CidSet {

15
include/ipfs/repo/fsrepo/journalstore.h
View File

@ -10,12 +10,17 @@
#include "libp2p/db/datastore.h"
struct JournalRecord {
unsigned long long timestamp;
int pin;
uint8_t *hash;
size_t hash_size;
unsigned long long timestamp; // the timestamp of the file
int pin; // true if it is to be stored, false if it is to be deleted
int pending; // true if we do not have this file yet
uint8_t *hash; // the hash of the block (file)
size_t hash_size; // the size of the hash
};
struct JournalRecord* lmdb_journal_record_new();
int lmdb_journal_record_free(struct JournalRecord* rec);
/**
* Open a cursor to the journalstore table
*/
@ -33,4 +38,4 @@ int repo_journalstore_cursor_close(struct Datastore* datastore, void* cursor);
int journal_record_free(struct JournalRecord* rec);
int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, unsigned long long timestamp, const uint8_t *hash, size_t hash_size);
int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, struct JournalRecord* record);

4
journal/journal.c
View File

@ -94,7 +94,7 @@ int ipfs_journal_free_records(struct Libp2pVector* records) {
if (records != NULL) {
for (int i = 0; i < records->total; i++) {
struct JournalRecord* rec = (struct JournalRecord*)libp2p_utils_vector_get(records, i);
journal_record_free(rec);
lmdb_journal_record_free(rec);
}
libp2p_utils_vector_free(records);
}
@ -319,7 +319,7 @@ int ipfs_journal_handle_message(const uint8_t* incoming, size_t incoming_size, s
case (JOURNAL_ENTRY_NEEDED): {
// go get a file
struct Block* block = NULL;
struct Cid* cid = ipfs_cid_new(0, curr->hash, curr->hash_size, CID_PROTOBUF);
struct Cid* cid = ipfs_cid_new(0, curr->hash, curr->hash_size, CID_DAG_PROTOBUF);
// debugging
char* str = NULL;
libp2p_logger_debug("journal", "Looking for block %s.\n", ipfs_cid_to_string(cid, &str));

2
repo/fsrepo/fs_repo.c
View File

@ -818,7 +818,7 @@ int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, s
libp2p_datastore_record_free(datastore_record);
// now get the block from the blockstore
struct Cid* cid = ipfs_cid_new(0, hash, hash_length, CID_PROTOBUF);
struct Cid* cid = ipfs_cid_new(0, hash, hash_length, CID_DAG_PROTOBUF);
if (cid == NULL)
return 0;
struct Blockstore* blockstore = ipfs_blockstore_new(fs_repo);

10
repo/fsrepo/lmdb_datastore.c
View File

@ -181,7 +181,15 @@ int repo_fsrepo_lmdb_put(unsigned const char* key, size_t key_size, unsigned cha
retVal = mdb_put(mdb_txn, mdb_dbi, &db_key, &db_value, MDB_NODUPDATA | MDB_NOOVERWRITE);
if (retVal == 0) {
// the normal case
lmdb_journalstore_journal_add(mdb_txn, timestamp, key, key_size);
// add it to the journalstore
struct JournalRecord* rec = lmdb_journal_record_new();
rec->hash = (uint8_t*)key;
rec->hash_size = key_size;
rec->timestamp = timestamp;
rec->pending = 1;
rec->pin = 1;
lmdb_journalstore_journal_add(mdb_txn, rec);
lmdb_journal_record_free(rec);
retVal = 1;
} else {
if (retVal == MDB_KEYEXIST) // We tried to add a key that already exists. Skip.

44
repo/fsrepo/lmdb_journalstore.c
View File

@ -7,8 +7,19 @@
#include "ipfs/repo/fsrepo/journalstore.h"
#include "ipfs/repo/fsrepo/lmdb_datastore.h"
struct JournalRecord* lmdb_journal_record_new() {
struct JournalRecord* rec = (struct JournalRecord*) malloc(sizeof(struct JournalRecord));
if (rec != NULL) {
rec->hash = NULL;
rec->hash_size = 0;
rec->pending = 0;
rec->pin = 0;
rec->timestamp = 0;
}
return rec;
}
int journal_record_free(struct JournalRecord* rec) {
int lmdb_journal_record_free(struct JournalRecord* rec) {
if (rec != NULL) {
if (rec->hash != NULL)
free(rec->hash);
@ -26,30 +37,26 @@ int journal_record_free(struct JournalRecord* rec) {
* @param hash_size the size of the hash
* @returns true(1) on success, false(0) otherwise
*/
int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, unsigned long long timestamp, const uint8_t *hash, size_t hash_size) {
int lmdb_journalstore_journal_add(MDB_txn* mdb_txn, struct JournalRecord* journal_record) {
MDB_dbi mdb_dbi;
struct MDB_val db_key;
struct MDB_val db_value;
// build the record, which is a timestamp as a key, a byte that is the pin flag, and the hash as the value
// build the record, which is a timestamp as a key
uint8_t time_varint[8];
size_t time_varint_size = 0;
varint_encode(timestamp, &time_varint[0], 8, &time_varint_size);
varint_encode(journal_record->timestamp, &time_varint[0], 8, &time_varint_size);
size_t record_size = hash_size + 1;
size_t record_size = journal_record->hash_size + 2;
uint8_t record[record_size];
record[0] = 1;
memcpy(&record[1], hash, hash_size);
// debug
size_t b58size = 100;
uint8_t *b58key = (uint8_t *) malloc(b58size);
libp2p_crypto_encoding_base58_encode(hash, hash_size, &b58key, &b58size);
libp2p_logger_debug("lmdb_journalstore", "Adding hash %s to journalstore.\n", b58key);
free(b58key);
// Field 1: pin flag
record[0] = journal_record->pin;
// Field 2: pending flag
record[1] = journal_record->pending;
// field 3: hash
memcpy(&record[2], journal_record->hash, journal_record->hash_size);
// open the journal table
if (mdb_dbi_open(mdb_txn, "JOURNALSTORE", MDB_DUPSORT | MDB_CREATE, &mdb_dbi) != 0) {
return 0;
}
@ -138,10 +145,13 @@ int repo_journalstore_cursor_get(struct Datastore* datastore, void* crsr, enum D
rec->timestamp = varint_decode(mdb_key.mv_data, mdb_key.mv_size, &varint_size);
// pin flag
rec->pin = ((uint8_t*)mdb_value.mv_data)[0];
rec->hash_size = mdb_value.mv_size - 1;
// pending flag
rec->pending = ((uint8_t*)mdb_value.mv_data)[1];
// hash
rec->hash_size = mdb_value.mv_size - 2;
rec->hash = malloc(rec->hash_size);
uint8_t *val = (uint8_t*)mdb_value.mv_data;
memcpy(rec->hash, &val[1], rec->hash_size);
memcpy(rec->hash, &val[2], rec->hash_size);
return 1;
}
return 0;

10
test/cid/test_cid.h
View File

@ -12,7 +12,7 @@ int test_cid_new_free() {
unsigned char* hash = (unsigned char*)"ABC123";
struct Cid* cid = ipfs_cid_new(0, hash, strlen((char*)hash), CID_PROTOBUF);
struct Cid* cid = ipfs_cid_new(0, hash, strlen((char*)hash), CID_DAG_PROTOBUF);
if (cid == NULL)
return 0;
@ -20,7 +20,7 @@ int test_cid_new_free() {
if (cid->version != 0)
return 0;
if (cid->codec != CID_PROTOBUF)
if (cid->codec != CID_DAG_PROTOBUF)
return 0;
if (cid->hash_length != strlen((char*)hash))
@ -62,7 +62,7 @@ int test_cid_cast_multihash() {
return 0;
if (cid.hash_length != 32)
return 0;
if (cid.codec != CID_PROTOBUF)
if (cid.codec != CID_DAG_PROTOBUF)
return 0;
if (strncmp((char*)hashed, (char*)cid.hash, 32) != 0)
return 0;
@ -85,7 +85,7 @@ int test_cid_cast_non_multihash() {
// first the version
array[0] = 0;
// then the codec
array[1] = CID_PROTOBUF;
array[1] = CID_DAG_PROTOBUF;
// then the hash
memcpy(&array[2], hashed, 32);
@ -99,7 +99,7 @@ int test_cid_cast_non_multihash() {
return 0;
if (cid.hash_length != 32)
return 0;
if (cid.codec != CID_PROTOBUF)
if (cid.codec != CID_DAG_PROTOBUF)
return 0;
if (strncmp((char*)hashed, (char*)cid.hash, 32) != 0)
return 0;

6
test/exchange/test_bitswap.h
View File

@ -136,7 +136,7 @@ int test_bitswap_retrieve_file()
ipfs_import_file(NULL, "/home/parallels/ipfstest/hello_world.txt", &node, localNode, &bytes_written, 0);
// build the Cid from the node information
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF);
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF);
if (cid == NULL)
goto exit;
@ -220,7 +220,7 @@ int test_bitswap_retrieve_file_remote() {
ipfs_node2->routing->Bootstrap(ipfs_node2->routing);
// this does the heavy lifting...
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF);
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF);
if (!ipfs_node2->exchange->GetBlock(ipfs_node2->exchange, cid, &result)) {
libp2p_logger_error("test_bitswap", "GetBlock returned false\n");
goto exit;
@ -445,7 +445,7 @@ int test_bitswap_retrieve_file_third_party() {
ipfs_node3->routing->Bootstrap(ipfs_node3->routing);
// this does the heavy lifting...
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_PROTOBUF);
cid = ipfs_cid_new(0, node->hash, node->hash_size, CID_DAG_PROTOBUF);
if (!ipfs_node3->exchange->GetBlock(ipfs_node3->exchange, cid, &result)) {
libp2p_logger_error("test_bitswap", "GetBlock returned false\n");
goto exit;

2
test/storage/test_blocks.h
View File

@ -24,7 +24,7 @@ int test_blocks_new() {
return 0;
}
if (block->cid->codec != CID_PROTOBUF) {
if (block->cid->codec != CID_DAG_PROTOBUF) {
ipfs_block_free(block);
return 0;
}

4
test/storage/test_datastore.h
View File

@ -94,14 +94,14 @@ int test_datastore_list_journal() {
enum DatastoreCursorOp op = CURSOR_FIRST;
do {
if (repo_journalstore_cursor_get(fs_repo->config->datastore, crsr, op, &record) == 0) {
journal_record_free(record);
lmdb_journal_record_free(record);
record = NULL;
}
// display record
libp2p_logger_debug("test_datastore", "Timestamp: %llu.\n", record->timestamp);
libp2p_logger_debug("test_datastore", "Pin: %s.\n", record->pin == 1 ? "Y" : "N");
// free record
journal_record_free(record);
lmdb_journal_record_free(record);
record = NULL;
op = CURSOR_NEXT;
} while (record != NULL);