jiacheng-wu / lipp Goto Github PK

int main(){
    LIPP<int , int> lipp{};
    for(int i=0;i<100000000;i++){
        lipp.insert(std::pair<int , int>(i,i));
    }
}

terminate called after throwing an instance of 'St9bad_alloc'
  what():  std::bad_alloc
zsh: abort (core dumped)  ./example_bulk_load

#include <lipp.h>
#include <iostream>
#include <vector>
#include <cassert>
#include <algorithm>
#include <iostream>
#include <fstream>

using namespace std;

template<class T>
bool load_binary_data(T data[], int length, const std::string &file_path) {
    std::ifstream is(file_path.c_str(), std::ios::binary | std::ios::in);
    if (!is.is_open()) {
        return false;
    }
    is.read(reinterpret_cast<char *>(data), std::streamsize(length * sizeof(T)));
    is.close();

    return true;
}

int main()
{
    LIPP<uint64_t, uint64_t> lipp;
    size_t data_size = 200000000;
    size_t bulk_load_size = 100000000;

    // prepare data
    uint64_t *keys = new uint64_t[data_size];
    load_binary_data(keys, data_size, "../test.data");

    vector<pair<uint64_t, uint64_t>> data;
    std::sort(keys, keys + bulk_load_size);
    auto last = std::unique(keys, keys + bulk_load_size);
    bulk_load_size = last - keys;
    for (size_t i = 0; i < bulk_load_size; i ++) {
        data.push_back({keys[i], keys[i]});
    }

    // bulk load
    lipp.bulk_load(data.data(), bulk_load_size);

    for(int i = 0; i < data_size - bulk_load_size; i++) {
        lipp.insert(keys[bulk_load_size + i], i);
    }
    
    return 0;
}

bachfischer@learned-indexes-benchmark:~/lipp$ ./build/experiments 
Keys loaded
Killed

Program received signal SIGSEGV, Segmentation fault.
0x00000000004048d1 in LIPP<unsigned long, unsigned long, true>::build_tree_bulk_fmcd(unsigned long*, unsigned long*, int) ()

#include <lipp.h>
#include <iostream>
#include <vector>
#include <cassert>
#include <algorithm>
#include <iostream>
#include <fstream>

using namespace std;

template<class T>
bool load_binary_data(T data[], int length, const std::string &file_path) {
    std::ifstream is(file_path.c_str(), std::ios::binary | std::ios::in);
    if (!is.is_open()) {
        return false;
    }
    is.read(reinterpret_cast<char *>(data), std::streamsize(length * sizeof(T)));
    is.close();

    return true;
}


template<typename T>
T *unique_data(T *key1, size_t &size1, T *key2, size_t &size2) {
    size_t ptr1 = 0;
    size_t ptr2 = 0;

    std::sort(key1, key1 + size1);
    size1 = std::unique(key1, key1 + size1) - key1;
    std::sort(key2, key2 + size2);
    size2 = std::unique(key2, key2 + size2) - key2;

    size_t result = 0;

    while (ptr1 < size1 && ptr2 < size2) {
        while (key1[ptr1] < key2[ptr2]) {
            ptr1++;
        }
        if (key1[ptr1] == key2[ptr2]) {
            ptr2++;
            continue;
        }
        key2[result++] = key2[ptr2++];
    }

    while (ptr2 < size2) {
        key2[result++] = key2[ptr2++];
    }

    size2 = result;
    std::random_shuffle(key2, key2 + size2);

    return &key2[result];
}

int main() {
    LIPP <uint64_t, uint64_t> lipp;
    size_t data_size = 200000000;
    size_t bulk_load_size = 100000000;
    size_t to_insert = data_size - bulk_load_size;

    // prepare data
    uint64_t *keys = new uint64_t[data_size];
    cout << "Loading data" << endl;
    load_binary_data(keys, bulk_load_size, "../fb_200M_uint64");
    load_binary_data(keys + bulk_load_size, to_insert, "../books_200M_uint64");
    cout << "Make data Unique" << endl;
    unique_data(keys, bulk_load_size, keys + bulk_load_size, to_insert);

    vector <pair<uint64_t, uint64_t>> data;
    cout << "bulk load size is " << bulk_load_size << endl;
    for (size_t i = 0; i < bulk_load_size; i++) {
        data.push_back({keys[i], keys[i]});
    }

    cout << "Begin to bulkload" << endl;

    // bulk load
    lipp.bulk_load(data.data(), bulk_load_size);

    cout << "Finish bulkload" << endl;
    cout << "Begin insert" << endl;
    for (int i = 0; i < to_insert; i++) {
        lipp.insert(keys[bulk_load_size + i], i);
    }

    cout << "Finish insert" << endl;

    return 0;
}

#0  0x0000000000411100 in LIPP<unsigned long, unsigned long, true>::insert_tree (this=0x7fffffff3140, _node=0xb5d098e0, key=@0x7fffc75b1a58: 772277387847372864, 
    value=@0x7fffffff3238: 73) at lipp/src/core/lipp.h:795
#1  0x000000000041015c in LIPP<unsigned long, unsigned long, true>::insert (this=0x7fffffff3140, key=@0x7fffc75b1a58: 772277387847372864, value=@0x7fffffff3238: 73)
    at lipp/src/core/lipp.h:110
#2  0x000000000040f55f in main () atlipp/src/examples/example_bulk_load.cpp:85