More work on persisting data to disk.
Blockstore now storing the data, whereas datastore is storing the key and filename. The key should be the multihash (currently the sha256, not the multihash), and the value is the filename (base32).
This commit is contained in:
jmjatlanta
parent
88d177cf4a
commit
033dd767b4
20 changed files with 564 additions and 57 deletions
117
blocks/block.c
117
blocks/block.c
|
|
@ -9,6 +9,100 @@
|
|||
#include "ipfs/blocks/block.h"
|
||||
#include "ipfs/cid/cid.h"
|
||||
|
||||
/***
|
||||
* The protobuf functions
|
||||
*/
|
||||
// protobuf fields: data & data_length cid
|
||||
enum WireType ipfs_block_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED};
|
||||
|
||||
/**
|
||||
* Determine the approximate size of an encoded block
|
||||
* @param block the block to measure
|
||||
* @returns the approximate size needed to encode the protobuf
|
||||
*/
|
||||
size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block) {
|
||||
return 22 + ipfs_cid_protobuf_encode_size(block->cid) + block->data_length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode the Block into protobuf format
|
||||
* @param block the block to encode
|
||||
* @param buffer the buffer to fill
|
||||
* @param max_buffer_size the max size of the buffer
|
||||
* @param bytes_written the number of bytes used
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
|
||||
// data & data_size
|
||||
size_t bytes_used = 0;
|
||||
*bytes_written = 0;
|
||||
int retVal = 0;
|
||||
retVal = protobuf_encode_length_delimited(1, ipfs_block_message_fields[0], (char*)block->data, block->data_length, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
|
||||
*bytes_written += bytes_used;
|
||||
// cid
|
||||
size_t cid_size = ipfs_cid_protobuf_encode_size(block->cid);
|
||||
unsigned char cid[cid_size];
|
||||
retVal = ipfs_cid_protobuf_encode(block->cid, cid, cid_size, &cid_size);
|
||||
retVal = protobuf_encode_length_delimited(2, ipfs_block_message_fields[1], (char*)cid, cid_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
|
||||
*bytes_written += bytes_used;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/***
|
||||
* Decode from a protobuf stream into a Block struct
|
||||
* @param buffer the buffer to pull from
|
||||
* @param buffer_length the length of the buffer
|
||||
* @param block the block to fill
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block) {
|
||||
size_t pos = 0;
|
||||
int retVal = 0;
|
||||
unsigned char* temp_buffer = NULL;
|
||||
size_t temp_size;
|
||||
|
||||
if (ipfs_blocks_block_new(block) == 0)
|
||||
goto exit;
|
||||
|
||||
while(pos < buffer_length) {
|
||||
size_t bytes_read = 0;
|
||||
int field_no;
|
||||
enum WireType field_type;
|
||||
if (protobuf_decode_field_and_type(&buffer[pos], buffer_length, &field_no, &field_type, &bytes_read) == 0) {
|
||||
goto exit;
|
||||
}
|
||||
pos += bytes_read;
|
||||
switch(field_no) {
|
||||
case (1): // data
|
||||
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*block)->data), &((*block)->data_length), &bytes_read) == 0)
|
||||
goto exit;
|
||||
pos += bytes_read;
|
||||
break;
|
||||
case (2): // cid
|
||||
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0)
|
||||
goto exit;
|
||||
pos += bytes_read;
|
||||
if (ipfs_cid_protobuf_decode(temp_buffer, temp_size, &((*block)->cid)) == 0)
|
||||
goto exit;
|
||||
free(temp_buffer);
|
||||
temp_buffer = NULL;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
retVal = 1;
|
||||
|
||||
exit:
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(*block);
|
||||
}
|
||||
if (temp_buffer != NULL)
|
||||
free(temp_buffer);
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
|
||||
/***
|
||||
* Create a new block based on the incoming data
|
||||
* @param data the data to base the block on
|
||||
|
|
@ -16,35 +110,38 @@
|
|||
* @param block a pointer to the struct Block that will be created
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block) {
|
||||
int ipfs_blocks_block_new(struct Block** block) {
|
||||
|
||||
// allocate memory for structure
|
||||
(*block) = (struct Block*)malloc(sizeof(struct Block));
|
||||
if ((*block) == NULL)
|
||||
return 0;
|
||||
(*block)->data = NULL;
|
||||
(*block)->data_length = 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block) {
|
||||
// cid
|
||||
unsigned char hash[32];
|
||||
if (libp2p_crypto_hashing_sha256(data, data_size, &hash[0]) == 0) {
|
||||
free(*block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &((*block)->cid)) == 0) {
|
||||
free(*block);
|
||||
if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &(block->cid)) == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
(*block)->data_length = data_size;
|
||||
block->data_length = data_size;
|
||||
|
||||
(*block)->data = malloc(sizeof(unsigned char) * data_size);
|
||||
if ( (*block)->data == NULL) {
|
||||
ipfs_cid_free((*block)->cid);
|
||||
free(*block);
|
||||
block->data = malloc(sizeof(unsigned char) * data_size);
|
||||
if ( block->data == NULL) {
|
||||
ipfs_cid_free(block->cid);
|
||||
return 0;
|
||||
}
|
||||
|
||||
memcpy( (*block)->data, data, data_size);
|
||||
memcpy( block->data, data, data_size);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include "ipfs/blocks/block.h"
|
||||
#include "ipfs/datastore/ds_helper.h"
|
||||
#include "ipfs/repo/fsrepo/fs_repo.h"
|
||||
#include "ipfs/os/utils.h"
|
||||
|
||||
/**
|
||||
* Delete a block based on its Cid
|
||||
|
|
@ -25,14 +26,59 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) {
|
||||
size_t key_length = libp2p_crypto_encoding_base32_encode_size(cid->hash_length);
|
||||
unsigned char* buffer = (unsigned char*)malloc(key_length + 1);
|
||||
if (buffer == NULL)
|
||||
return NULL;
|
||||
int retVal = ipfs_datastore_helper_ds_key_from_binary(cid->hash, cid->hash_length, &buffer[0], key_length, &key_length);
|
||||
if (retVal == 0) {
|
||||
free(buffer);
|
||||
return NULL;
|
||||
}
|
||||
buffer[key_length] = 0;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
char* ipfs_blockstore_path_get(const struct FSRepo* fs_repo, const char* filename) {
|
||||
int filepath_size = strlen(fs_repo->path) + 12;
|
||||
char filepath[filepath_size];
|
||||
int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", filepath, filepath_size);
|
||||
if (retVal == 0) {
|
||||
free(filepath);
|
||||
return 0;
|
||||
}
|
||||
int complete_filename_size = strlen(filepath) + strlen(filename) + 2;
|
||||
char* complete_filename = (char*)malloc(complete_filename_size);
|
||||
retVal = os_utils_filepath_join(filepath, filename, complete_filename, complete_filename_size);
|
||||
return complete_filename;
|
||||
}
|
||||
|
||||
/***
|
||||
* Find a block based on its Cid
|
||||
* @param cid the Cid to look for
|
||||
* @param block where to put the data to be returned
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo) {
|
||||
return 0;
|
||||
int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) {
|
||||
// get datastore key, which is a base32 key of the multihash
|
||||
unsigned char* key = ipfs_blockstore_cid_to_base32(cid);
|
||||
|
||||
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
|
||||
|
||||
size_t file_size = os_utils_file_size(filename);
|
||||
unsigned char buffer[file_size];
|
||||
|
||||
FILE* file = fopen(filename, "rb");
|
||||
size_t bytes_read = fread(buffer, 1, file_size, file);
|
||||
fclose(file);
|
||||
|
||||
int retVal = ipfs_blocks_block_protobuf_decode(buffer, bytes_read, block);
|
||||
|
||||
free(key);
|
||||
free(filename);
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
/***
|
||||
|
|
@ -42,14 +88,46 @@ int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_
|
|||
*/
|
||||
int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) {
|
||||
// from blockstore.go line 118
|
||||
// Get Datastore key, which is a base32 key of the binary,
|
||||
size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length);
|
||||
unsigned char key[key_length];
|
||||
int retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
int retVal = 0;
|
||||
|
||||
// send to Put with key
|
||||
fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
|
||||
// Get Datastore key, which is a base32 key of the multihash,
|
||||
unsigned char* key = ipfs_blockstore_cid_to_base32(block->cid);
|
||||
if (key == NULL) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//TODO: put this in subdirectories
|
||||
|
||||
// turn the block into a binary array
|
||||
size_t protobuf_len = ipfs_blocks_block_protobuf_encode_size(block);
|
||||
unsigned char protobuf[protobuf_len];
|
||||
retVal = ipfs_blocks_block_protobuf_encode(block, protobuf, protobuf_len, &protobuf_len);
|
||||
if (retVal == 0) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// now write byte array to file
|
||||
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
|
||||
if (filename == NULL) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
FILE* file = fopen(filename, "wb");
|
||||
int bytes_written = fwrite(protobuf, 1, protobuf_len, file);
|
||||
fclose(file);
|
||||
if (bytes_written != protobuf_len) {
|
||||
free(key);
|
||||
free(filename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// send to Put with key (this is now done separately)
|
||||
//fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
|
||||
|
||||
free(key);
|
||||
free(filename);
|
||||
return 1;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue