agorise/c-ipfs
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity
c-ipfs/repo/fsrepo/fs_repo.c
jmjatlanta c8fdb084e4 Moving closer to binary compatability with go version of ipfs 
The files are now stored in the same format. Now it is necessary to
reverse engineer the directory structure
2016-12-21 06:40:19 -05:00

571 lines
19 KiB
C
Raw Blame History

#include <stdio.h>
#include <sys/stat.h>
#include "ipfs/blocks/blockstore.h"
#include "libp2p/crypto/encoding/base64.h"
#include "ipfs/datastore/ds_helper.h"
#include "ipfs/repo/config/datastore.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/os/utils.h"
#include "ipfs/repo/fsrepo/lmdb_datastore.h"
#include "jsmn.h"
/**
* private methods
*/
/**
* writes the config file
* @param full_filename the full filename of the config file in the OS
* @param config the details to put into the file
* @returns true(1) on success, else false(0)
*/
int repo_config_write_config_file(char* full_filename, struct RepoConfig* config) {
FILE* out_file = fopen(full_filename, "w");
if (out_file == NULL)
return 0;
fprintf(out_file, "{\n");
fprintf(out_file, " \"Identity\": {\n");
fprintf(out_file, " \"PeerID\": \"%s\",\n", config->identity->peer_id);
// print correct format of private key
// first base 64 it
size_t encoded_size = libp2p_crypto_encoding_base64_encode_size(config->identity->private_key.der_length);
unsigned char encoded_buffer[encoded_size + 1];
int retVal = libp2p_crypto_encoding_base64_encode((unsigned char*)config->identity->private_key.der, config->identity->private_key.der_length, encoded_buffer, encoded_size, &encoded_size);
if (retVal == 0)
return 0;
encoded_buffer[encoded_size] = 0;
fprintf(out_file, " \"PrivKey\": \"%s\"\n", encoded_buffer);
fprintf(out_file, " },\n");
fprintf(out_file, " \"Datastore\": {\n");
fprintf(out_file, " \"Type\": \"%s\",\n", config->datastore->type);
fprintf(out_file, " \"Path\": \"%s\",\n", config->datastore->path);
fprintf(out_file, " \"StorageMax\": \"%s\",\n", config->datastore->storage_max);
fprintf(out_file, " \"StorageGCWatermark\": %d,\n", config->datastore->storage_gc_watermark);
fprintf(out_file, " \"GCPeriod\": \"%s\",\n", config->datastore->gc_period);
fprintf(out_file, " \"Params\": null,\n");
fprintf(out_file, " \"NoSync\": %s,\n", config->datastore->no_sync ? "true" : "false");
fprintf(out_file, " \"HashOnRead\": %s,\n", config->datastore->hash_on_read ? "true" : "false");
fprintf(out_file, " \"BloomFilterSize\": %d\n", config->datastore->bloom_filter_size);
fprintf(out_file, " },\n \"Addresses\": {\n");
fprintf(out_file, " \"Swarm\": [\n");
for(int i = 0; i < config->addresses->swarm->num_addresses; i++) {
fprintf(out_file, " \"%s\"", config->addresses->swarm->addresses[i]);
if (i != (config->addresses->swarm->num_addresses - 1))
fprintf(out_file, ",\n");
else
fprintf(out_file, "\n");
}
fprintf(out_file, " ],\n");
fprintf(out_file, " \"API\": \"%s\",\n", config->addresses->api);
fprintf(out_file, " \"Gateway\": \"%s\"\n", config->addresses->gateway);
fprintf(out_file, " },\n \"Mounts\": {\n");
fprintf(out_file, " \"IPFS\": \"%s\",\n", config->mounts.ipfs);
fprintf(out_file, " \"IPNS\": \"%s\",\n", config->mounts.ipns);
fprintf(out_file, " \"FuseAllowOther\": %s\n", "false");
fprintf(out_file, " },\n \"Discovery\": {\n \"MDNS\": {\n");
fprintf(out_file, " \"Enabled\": %s,\n", config->discovery.mdns.enabled ? "true" : "false");
fprintf(out_file, " \"Interval\": %d\n }\n },\n", config->discovery.mdns.interval);
fprintf(out_file, " \"Ipns\": {\n");
fprintf(out_file, " \"RepublishedPeriod\": \"\",\n");
fprintf(out_file, " \"RecordLifetime\": \"\",\n");
fprintf(out_file, " \"ResolveCacheSize\": %d\n", config->ipns.resolve_cache_size);
fprintf(out_file, " },\n \"Bootstrap\": [\n");
/*
for(int i = 0; i < config->peer_addresses.num_peers; i++) {
struct IPFSAddr* peer = config->peer_addresses.peers[i];
fprintf(out_file, " \"%s\"", peer->entire_string);
if (i < config->peer_addresses.num_peers - 1)
fprintf(out_file, ",\n");
else
fprintf(out_file, "\n");
}
*/
fprintf(out_file, " ],\n \"Tour\": {\n \"Last\": \"\"\n },\n");
fprintf(out_file, " \"Gateway\": {\n");
fprintf(out_file, " \"HTTPHeaders\": {\n");
for (int i = 0; i < config->gateway->http_headers->num_elements; i++) {
fprintf(out_file, " \"%s\": [\n \"%s\"\n ]", config->gateway->http_headers->headers[i]->header, config->gateway->http_headers->headers[i]->value);
if (i < config->gateway->http_headers->num_elements - 1)
fprintf(out_file, ",\n");
else
fprintf(out_file, "\n },\n");
}
fprintf(out_file, " \"RootRedirect\": \"%s\"\n", config->gateway->root_redirect);
fprintf(out_file, " \"Writable\": %s\n", config->gateway->writable ? "true" : "false");
fprintf(out_file, " \"PathPrefixes\": []\n");
fprintf(out_file, " },\n \"SupernodeRouting\": {\n");
fprintf(out_file, " \"Servers\": null\n },");
fprintf(out_file, " \"API\": {\n \"HTTPHeaders\": null\n },\n");
fprintf(out_file, " \"Swarm\": {\n \"AddrFilters\": null\n }\n}");
fclose(out_file);
return 1;
}
/**
* constructs the FSRepo struct.
* Remember: ipfs_repo_fsrepo_free must be called
* @param repo_path the path to the repo
* @param config the optional config file. NOTE: if passed, fsrepo_free will free resources of the RepoConfig.
* @param repo the struct to allocate memory for
* @returns false(0) if something bad happened, otherwise true(1)
*/
int ipfs_repo_fsrepo_new(const char* repo_path, struct RepoConfig* config, struct FSRepo** repo) {
*repo = (struct FSRepo*)malloc(sizeof(struct FSRepo));
if (repo_path == NULL) {
// get the user's home directory
char* home_dir = os_utils_get_homedir();
char* default_subdir = "/.ipfs";
unsigned long newPathLen = strlen(home_dir) + strlen(default_subdir) + 2; // 1 for slash and 1 for end
(*repo)->path = malloc(sizeof(char) * newPathLen);
if ((*repo)->path == NULL) {
free( (*repo));
return 0;
}
os_utils_filepath_join(os_utils_get_homedir(), default_subdir, (*repo)->path, newPathLen);
} else {
int len = strlen(repo_path) + 1;
(*repo)->path = (char*)malloc(len);
strncpy((*repo)->path, repo_path, len);
}
// allocate other structures
if (config != NULL)
(*repo)->config = config;
else {
if (ipfs_repo_config_new(&((*repo)->config)) == 0) {
free((*repo)->path);
return 0;
}
}
return 1;
}
/**
* Cleans up memory
* @param repo the struct to clean up
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_free(struct FSRepo* repo) {
if (repo != NULL) {
if (repo->path != NULL)
free(repo->path);
if (repo->config != NULL)
ipfs_repo_config_free(repo->config);
free(repo);
}
return 1;
}
/**
* checks to see if the repo is initialized at the given path
* @param full_path the path to the repo
* @returns true(1) if the config file is there, false(0) otherwise
*/
int repo_config_is_initialized(char* full_path) {
char* config_file_full_path;
int retVal = repo_config_get_file_name(full_path, &config_file_full_path);
if (!retVal)
return 0;
if (os_utils_file_exists(config_file_full_path))
retVal = 1;
else
retVal = 0;
free(config_file_full_path);
return retVal;
}
/***
* Check to see if the repo is initialized
* @param full_path the path to the repo
* @returns true(1) if it is initialized, false(0) otherwise.
*/
int fs_repo_is_initialized_unsynced(char* full_path) {
return repo_config_is_initialized(full_path);
}
/**
* checks to see if the repo is initialized
* @param full_path the full path to the repo
* @returns true(1) if it is initialized, otherwise false(0)
*/
int repo_check_initialized(char* full_path) {
// note the old version of this reported an error if the repo was a .go-ipfs repo (by looking at the path)
// this version skips that step
return fs_repo_is_initialized_unsynced(full_path);
}
/***
* Reads the file, placing its contents in buffer
* NOTE: this allocates memory for buffer, and should be freed
* @param path the path to the config file
* @param buffer where to put the contents
* @returns true(1) on success
*/
int _read_file(const char* path, char** buffer) {
int file_size = os_utils_file_size(path);
if (file_size <= 0)
return 0;
// allocate memory
*buffer = malloc(file_size + 1);
if (*buffer == NULL) {
return 0;
}
memset(*buffer, 0, file_size + 1);
// open file
FILE* in_file = fopen(path, "r");
// read data
fread(*buffer, file_size, 1, in_file);
// cleanup
fclose(in_file);
return 1;
}
/**
* Find the position of a key
* @param data the string that contains the json
* @param tokens the tokens of the parsed string
* @param tok_length the number of tokens there are
* @param tag what we're looking for
* @returns the position of the requested token in the array, or -1
*/
int _find_token(const char* data, const jsmntok_t* tokens, int tok_length, int start_from, const char* tag) {
for(int i = start_from; i < tok_length; i++) {
jsmntok_t curr_token = tokens[i];
if ( curr_token.type == JSMN_STRING) {
// convert to string
int str_len = curr_token.end - curr_token.start;
char str[str_len + 1];
strncpy(str, &data[curr_token.start], str_len );
str[str_len] = 0;
if (strcmp(str, tag) == 0)
return i;
}
}
return -1;
}
/**
* Retrieves the value of a key / value pair from the JSON data
* @param data the full JSON string
* @param tokens the array of tokens
* @param tok_length the number of tokens
* @param search_from start search from this token onward
* @param tag what to search for
* @param result where to put the result. NOTE: allocates memory that must be freed
* @returns true(1) on success
*/
int _get_json_string_value(char* data, const jsmntok_t* tokens, int tok_length, int search_from, const char* tag, char** result) {
int pos = _find_token(data, tokens, tok_length, search_from, tag);
if (pos < 0)
return 0;
jsmntok_t curr_token = tokens[pos+1];
if (curr_token.type == JSMN_PRIMITIVE) {
// a null
*result = NULL;
}
if (curr_token.type != JSMN_STRING)
return 0;
// allocate memory
int str_len = curr_token.end - curr_token.start;
*result = malloc(sizeof(char) * str_len + 1);
if (*result == NULL)
return 0;
// copy in the string
strncpy(*result, &data[curr_token.start], str_len);
(*result)[str_len] = 0;
return 1;
}
/**
* Retrieves the value of a key / value pair from the JSON data
* @param data the full JSON string
* @param tokens the array of tokens
* @param tok_length the number of tokens
* @param search_from start search from this token onward
* @param tag what to search for
* @param result where to put the result
* @returns true(1) on success
*/
int _get_json_int_value(char* data, const jsmntok_t* tokens, int tok_length, int search_from, const char* tag, int* result) {
int pos = _find_token(data, tokens, tok_length, search_from, tag);
if (pos < 0)
return 0;
jsmntok_t curr_token = tokens[pos+1];
if (curr_token.type != JSMN_PRIMITIVE)
return 0;
// allocate memory
int str_len = curr_token.end - curr_token.start;
char str[str_len + 1];
// copy in the string
strncpy(str, &data[curr_token.start], str_len);
str[str_len] = 0;
if (strcmp(str, "true") == 0)
*result = 1;
else if (strcmp(str, "false") == 0)
*result = 0;
else if (strcmp(str, "null") == 0) // what should we do here?
*result = 0;
else // its a real number
*result = atoi(str);
return 1;
}
/***
* Opens the config file and puts the data into the FSRepo struct
* @param repo the FSRepo struct
* @returns 0 on failure, otherwise 1
*/
int fs_repo_open_config(struct FSRepo* repo) {
int retVal;
char* data;
size_t full_filename_length = strlen(repo->path) + 8;
char full_filename[full_filename_length];
retVal = os_utils_filepath_join(repo->path, "config", full_filename, full_filename_length);
if (retVal == 0)
return 0;
retVal = _read_file(full_filename, &data);
// parse the data
jsmn_parser parser;
jsmn_init(&parser);
int num_tokens = 256;
jsmntok_t tokens[num_tokens];
num_tokens = jsmn_parse(&parser, data, strlen(data), tokens, 256);
if (num_tokens <= 0) {
free(data);
return 0;
}
// fill FSRepo struct
// allocation done by fsrepo_new... repo->config = malloc(sizeof(struct RepoConfig));
// Identity
int curr_pos = _find_token(data, tokens, num_tokens, 0, "Identity");
if (curr_pos < 0) {
free(data);
return 0;
}
// the next should be the array, then string "PeerID"
_get_json_string_value(data, tokens, num_tokens, curr_pos, "PeerID", &repo->config->identity->peer_id);
char* priv_key_base64;
// then PrivKey
_get_json_string_value(data, tokens, num_tokens, curr_pos, "PrivKey", &priv_key_base64);
retVal = repo_config_identity_build_private_key(repo->config->identity, priv_key_base64);
if (retVal == 0) {
free(data);
free(priv_key_base64);
return 0;
}
// now the datastore
//int datastore_position = _find_token(data, tokens, num_tokens, 0, "Datastore");
_get_json_string_value(data, tokens, num_tokens, curr_pos, "Type", &repo->config->datastore->type);
_get_json_string_value(data, tokens, num_tokens, curr_pos, "Path", &repo->config->datastore->path);
_get_json_string_value(data, tokens, num_tokens, curr_pos, "StorageMax", &repo->config->datastore->storage_max);
_get_json_int_value(data, tokens, num_tokens, curr_pos, "StorageGCWatermark", &repo->config->datastore->storage_gc_watermark);
_get_json_string_value(data, tokens, num_tokens, curr_pos, "GCPeriod", &repo->config->datastore->gc_period);
_get_json_string_value(data, tokens, num_tokens, curr_pos, "Params", &repo->config->datastore->params);
_get_json_int_value(data, tokens, num_tokens, curr_pos, "NoSync", &repo->config->datastore->no_sync);
_get_json_int_value(data, tokens, num_tokens, curr_pos, "HashOnRead", &repo->config->datastore->hash_on_read);
_get_json_int_value(data, tokens, num_tokens, curr_pos, "BloomFilterSize", &repo->config->datastore->bloom_filter_size);
// free the memory used reading the json file
free(data);
free(priv_key_base64);
return 1;
}
/***
* set function pointers in the datastore struct to lmdb
* @param repo contains the information
* @returns true(1) on success
*/
int fs_repo_setup_lmdb_datastore(struct FSRepo* repo) {
return repo_fsrepo_lmdb_cast(repo->config->datastore);
}
/***
* opens the repo's datastore, and puts a reference to it in the FSRepo struct
* @param repo the FSRepo struct
* @returns 0 on failure, otherwise 1
*/
int fs_repo_open_datastore(struct FSRepo* repo) {
int argc = 0;
char** argv = NULL;
if (strncmp(repo->config->datastore->type, "lmdb", 4) == 0) {
// this is a LightningDB. Open it.
int retVal = fs_repo_setup_lmdb_datastore(repo);
if (retVal == 0)
return 0;
} else {
// add new datastore types here
return 0;
}
int retVal = repo->config->datastore->datastore_open(argc, argv, repo->config->datastore);
// do specific datastore cleanup here if needed
return retVal;
}
/**
* public methods
*/
/**
* opens a fsrepo
* @param repo the repo struct. Should contain the path. This method will do the rest
* @return 0 if there was a problem, otherwise 1
*/
int ipfs_repo_fsrepo_open(struct FSRepo* repo) {
//TODO: lock
// check if initialized
if (!repo_check_initialized(repo->path)) {
return 0;
}
//TODO: lock the file (remember to unlock)
//TODO: check the version, and make sure it is correct
//TODO: make sure the directory is writable
//TODO: open the config
if (!fs_repo_open_config(repo)) {
return 0;
}
// open the datastore
if (!fs_repo_open_datastore(repo)) {
return 0;
}
return 1;
}
/***
* checks to see if the repo is initialized
* @param repo_path the path to the repo
* @returns true(1) if it is initialized, otherwise false(0)
*/
int fs_repo_is_initialized(char* repo_path) {
//TODO: lock things up so that someone doesn't try an init or remove while this call is in progress
// don't forget to unlock
return fs_repo_is_initialized_unsynced(repo_path);
}
int ipfs_repo_fsrepo_datastore_init(struct FSRepo* fs_repo) {
// make the directory
if (repo_fsrepo_lmdb_create_directory(fs_repo->config->datastore) == 0)
return 0;
// fill in the function prototypes
return repo_fsrepo_lmdb_cast(fs_repo->config->datastore);
}
int ipfs_repo_fsrepo_blockstore_init(const struct FSRepo* fs_repo) {
size_t full_path_size = strlen(fs_repo->path) + 15;
char full_path[full_path_size];
int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", full_path, full_path_size);
if (retVal == 0)
return 0;
if (mkdir(full_path, S_IRWXU) != 0)
return 0;
return 1;
}
/**
* Initializes a new FSRepo at the given path with the provided config
* @param path the path to use
* @param config the information for the config file
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_init(struct FSRepo* repo) {
// TODO: Do a lock so 2 don't do this at the same time
// return error if this has already been done
if (fs_repo_is_initialized_unsynced(repo->path))
return 0;
int retVal = fs_repo_write_config_file(repo->path, repo->config);
if (retVal == 0)
return 0;
retVal = ipfs_repo_fsrepo_datastore_init(repo);
if (retVal == 0)
return 0;
retVal = ipfs_repo_fsrepo_blockstore_init(repo);
if (retVal == 0)
return 0;
// write the version to a file for migrations (see repo/fsrepo/migrations/mfsr.go)
//TODO: mfsr.RepoPath(repo_path).WriteVersion(RepoVersion)
return 1;
}
/**
* write the config file to disk
* @param path the path to the file
* @param config the config structure
* @returns true(1) on success
*/
int fs_repo_write_config_file(char* path, struct RepoConfig* config) {
if (fs_repo_is_initialized(path))
return 0;
char* buff = NULL;
if (!repo_config_get_file_name(path, &buff))
return 0;
int retVal = repo_config_write_config_file(buff, config);
free(buff);
return retVal;
}
/***
* Write a block to the datastore and blockstore
* @param block the block to write
* @param fs_repo the repo to write to
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo) {
/**
* What is put in the blockstore is the block.
* What is put in the datastore is the multihash (the Cid) as the key,
* and the base32 encoded multihash as the value.
*/
int retVal = 1;
retVal = ipfs_blockstore_put(block, fs_repo);
if (retVal == 0)
return 0;
// take the cid, base32 it, and send both to the datastore
size_t fs_key_length = 100;
unsigned char fs_key[fs_key_length];
retVal = ipfs_datastore_helper_ds_key_from_binary(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, &fs_key_length);
if (retVal == 0)
return 0;
retVal = fs_repo->config->datastore->datastore_put(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, fs_repo->config->datastore);
if (retVal == 0)
return 0;
return 1;
}
int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo) {
int retVal = 0;
// get the base32 hash from the database
// We do this only to see if it is in the database
size_t fs_key_length = 100;
unsigned char fs_key[fs_key_length];
retVal = fs_repo->config->datastore->datastore_get((const char*)hash, hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore);
if (retVal == 0) // maybe it doesn't exist?
return 0;
// now get the block from the blockstore
retVal = ipfs_blockstore_get(hash, hash_length, block, fs_repo);
return retVal;
}
Reference in a new issue View git blame Copy permalink