darma-tasking / magistrate Goto Github PK

View Code? Open in Web Editor NEW

12.0 3.0 4.0 1.56 MB

DARMA/magistrate => Serialization and checkpointing library

License: Other

CMake 3.86% C++ 90.43% C 1.35% Shell 1.76% Perl 0.52% Dockerfile 2.07%

hpc serialization checkpoint concurrency parallelism

magistrate's People

Contributors

Stargazers

Watchers

Forkers

philmiller rppawlo matthew-whitlock dalg24

magistrate's Issues

Fix warning in std::vector<bool> serialize code

In file included from /Users/jliffla/codes/vt/checkpoint/src/serdes_headers.h:58,
                 from /Users/jliffla/codes/vt/checkpoint/src/serialization_library_headers.h:48,
                 from /Users/jliffla/codes/vt/checkpoint/tests/unit/test_vector_bool_serialize.cc:47:
/Users/jliffla/codes/vt/checkpoint/src/container/vector_serialize.h: In instantiation of 'void serdes::serialize(Serializer&, std::vector<bool, VectorAllocator>&) [with Serializer = serdes::Sizer; VectorAllocator = std::allocator<bool>]':
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:238:16:   required from 'void serdes::SerializerDispatchNonByte<SerializerT, T>::apply(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U>*) [with U = std::vector<bool>; SerializerT = serdes::Sizer; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U> = std::vector<bool>]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:161:29:   required from 'void serdes::SerializerDispatchNonByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType) [with U = std::vector<bool>; SerializerT = serdes::Sizer; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_byte.h:108:13:   required from 'void serdes::SerializerDispatchByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U>*) [with U = std::vector<bool>; SerializerT = serdes::Sizer; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U> = std::vector<bool>]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:61:5:   required from 'static serdes::SerialSizeType serdes::Dispatch<T>::sizeType(T&) [with T = std::vector<bool>; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:217:46:   required from 'serdes::SerializedReturnType serdes::serializeType(T&, serdes::BufferObtainFnType) [with T = std::vector<bool>; serdes::SerializedReturnType = std::tuple<std::unique_ptr<serdes::Buffer, std::default_delete<serdes::Buffer> >, long unsigned int>; serdes::BufferObtainFnType = std::function<char*(long unsigned int)>]'
/Users/jliffla/codes/vt/checkpoint/src/serialize_interface.impl.h:59:40:   required from 'serialization::interface::SerializedReturnType serialization::interface::serialize(T&, serialization::interface::BufferCallbackType) [with T = std::vector<bool>; serialization::interface::SerializedReturnType = std::unique_ptr<serialization::interface::SerializedInfo>; serialization::interface::BufferCallbackType = std::function<char*(long unsigned int)>]'
/Users/jliffla/codes/vt/checkpoint/tests/unit/test_vector_bool_serialize.cc:54:81:   required from here
/Users/jliffla/codes/vt/checkpoint/src/container/vector_serialize.h:77:24: warning: comparison of integer expressions of different signedness: 'int' and 'std::vector<bool>::size_type' {aka 'long unsigned int'} [-Wsign-compare]
     for (auto i = 0; i < vec.size(); ++i) {
/Users/jliffla/codes/vt/checkpoint/src/container/vector_serialize.h: In instantiation of 'void serdes::serialize(Serializer&, std::vector<bool, VectorAllocator>&) [with Serializer = serdes::Unpacker; VectorAllocator = std::allocator<bool>]':
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:238:16:   required from 'void serdes::SerializerDispatchNonByte<SerializerT, T>::apply(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U>*) [with U = std::vector<bool>; SerializerT = serdes::Unpacker; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U> = std::vector<bool>]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:161:29:   required from 'void serdes::SerializerDispatchNonByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType) [with U = std::vector<bool>; SerializerT = serdes::Unpacker; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_byte.h:108:13:   required from 'void serdes::SerializerDispatchByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U>*) [with U = std::vector<bool>; SerializerT = serdes::Unpacker; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U> = std::vector<bool>]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:104:7:   required from 'static T& serdes::Dispatch<T>::unpackType(serdes::SerialByteType*, serdes::SerialByteType*, const SerialSizeType&, bool) [with T = std::vector<bool>; serdes::SerialByteType = char; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:253:36:   required from 'T* serdes::deserializeType(serdes::SerialByteType*, const SerialSizeType&, T*) [with T = std::vector<bool>; serdes::SerialByteType = char; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/serialize_interface.impl.h:69:38:   required from 'T* serialization::interface::deserialize(serialization::interface::SerialByteType*, serialization::interface::SizeType, T*) [with T = std::vector<bool>; serialization::interface::SerialByteType = char; serialization::interface::SizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/tests/unit/test_vector_bool_serialize.cc:62:3:   required from here
/Users/jliffla/codes/vt/checkpoint/src/container/vector_serialize.h:77:24: warning: comparison of integer expressions of different signedness: 'int' and 'std::vector<bool>::size_type' {aka 'long unsigned int'} [-Wsign-compare]
/Users/jliffla/codes/vt/checkpoint/src/container/vector_serialize.h: In instantiation of 'void serdes::serialize(Serializer&, std::vector<bool, VectorAllocator>&) [with Serializer = serdes::PackerBuffer<serdes::ManagedBuffer>; VectorAllocator = std::allocator<bool>]':
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:238:16:   required from 'void serdes::SerializerDispatchNonByte<SerializerT, T>::apply(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U>*) [with U = std::vector<bool>; SerializerT = serdes::PackerBuffer<serdes::ManagedBuffer>; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U> = std::vector<bool>]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:161:29:   required from 'void serdes::SerializerDispatchNonByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType) [with U = std::vector<bool>; SerializerT = serdes::PackerBuffer<serdes::ManagedBuffer>; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_byte.h:108:13:   required from 'void serdes::SerializerDispatchByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U>*) [with U = std::vector<bool>; SerializerT = serdes::PackerBuffer<serdes::ManagedBuffer>; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U> = std::vector<bool>]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:75:5:   required from 'static serdes::BufferPtrType serdes::Dispatch<T>::packTypeWithPacker(PackerT&, T&, const SerialSizeType&) [with PackerT = serdes::PackerBuffer<serdes::ManagedBuffer>; T = std::vector<bool>; serdes::BufferPtrType = std::unique_ptr<serdes::Buffer>; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:85:30:   required from 'static serdes::BufferPtrType serdes::Dispatch<T>::packType(T&, const SerialSizeType&, serdes::SerialByteType*) [with T = std::vector<bool>; serdes::BufferPtrType = std::unique_ptr<serdes::Buffer>; serdes::SerialSizeType = long unsigned int; serdes::SerialByteType = char]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:220:39:   required from 'serdes::SerializedReturnType serdes::serializeType(T&, serdes::BufferObtainFnType) [with T = std::vector<bool>; serdes::SerializedReturnType = std::tuple<std::unique_ptr<serdes::Buffer, std::default_delete<serdes::Buffer> >, long unsigned int>; serdes::BufferObtainFnType = std::function<char*(long unsigned int)>]'
/Users/jliffla/codes/vt/checkpoint/src/serialize_interface.impl.h:59:40:   required from 'serialization::interface::SerializedReturnType serialization::interface::serialize(T&, serialization::interface::BufferCallbackType) [with T = std::vector<bool>; serialization::interface::SerializedReturnType = std::unique_ptr<serialization::interface::SerializedInfo>; serialization::interface::BufferCallbackType = std::function<char*(long unsigned int)>]'
/Users/jliffla/codes/vt/checkpoint/tests/unit/test_vector_bool_serialize.cc:54:81:   required from here
/Users/jliffla/codes/vt/checkpoint/src/container/vector_serialize.h:77:24: warning: comparison of integer expressions of different signedness: 'int' and 'std::vector<bool>::size_type' {aka 'long unsigned int'} [-Wsign-compare]
/Users/jliffla/codes/vt/checkpoint/src/container/vector_serialize.h: In instantiation of 'void serdes::serialize(Serializer&, std::vector<bool, VectorAllocator>&) [with Serializer = serdes::PackerBuffer<serdes::UserBuffer>; VectorAllocator = std::allocator<bool>]':
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:238:16:   required from 'void serdes::SerializerDispatchNonByte<SerializerT, T>::apply(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U>*) [with U = std::vector<bool>; SerializerT = serdes::PackerBuffer<serdes::UserBuffer>; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U> = std::vector<bool>]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:161:29:   required from 'void serdes::SerializerDispatchNonByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType) [with U = std::vector<bool>; SerializerT = serdes::PackerBuffer<serdes::UserBuffer>; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch_serializer_byte.h:108:13:   required from 'void serdes::SerializerDispatchByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U>*) [with U = std::vector<bool>; SerializerT = serdes::PackerBuffer<serdes::UserBuffer>; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U> = std::vector<bool>]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:75:5:   required from 'static serdes::BufferPtrType serdes::Dispatch<T>::packTypeWithPacker(PackerT&, T&, const SerialSizeType&) [with PackerT = serdes::PackerBuffer<serdes::UserBuffer>; T = std::vector<bool>; serdes::BufferPtrType = std::unique_ptr<serdes::Buffer>; serdes::SerialSizeType = long unsigned int]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:88:30:   required from 'static serdes::BufferPtrType serdes::Dispatch<T>::packType(T&, const SerialSizeType&, serdes::SerialByteType*) [with T = std::vector<bool>; serdes::BufferPtrType = std::unique_ptr<serdes::Buffer>; serdes::SerialSizeType = long unsigned int; serdes::SerialByteType = char]'
/Users/jliffla/codes/vt/checkpoint/src/dispatch/dispatch.impl.h:220:39:   required from 'serdes::SerializedReturnType serdes::serializeType(T&, serdes::BufferObtainFnType) [with T = std::vector<bool>; serdes::SerializedReturnType = std::tuple<std::unique_ptr<serdes::Buffer, std::default_delete<serdes::Buffer> >, long unsigned int>; serdes::BufferObtainFnType = std::function<char*(long unsigned int)>]'
/Users/jliffla/codes/vt/checkpoint/src/serialize_interface.impl.h:59:40:   required from 'serialization::interface::SerializedReturnType serialization::interface::serialize(T&, serialization::interface::BufferCallbackType) [with T = std::vector<bool>; serialization::interface::SerializedReturnType = std::unique_ptr<serialization::interface::SerializedInfo>; serialization::interface::BufferCallbackType = std::function<char*(long unsigned int)>]'
/Users/jliffla/codes/vt/checkpoint/tests/unit/test_vector_bool_serialize.cc:54:81:   required from here
/Users/jliffla/codes/vt/checkpoint/src/container/vector_serialize.h:77:24: warning: comparison of integer expressions of different signedness: 'int' and 'std::vector<bool>::size_type' {aka 'long unsigned int'} [-Wsign-compare]

Automatically call parent-class serializers

Requiring Child::serialize() methods to call the corresponding Parent::serialize() is error-prone and hard to detect when it's missing. Automate this wherever possible.

Write serializer for Kokkos::DynRankView

Kokkos::DynamicView bug in serialization

There is a problem reconstructing with a given chunk and allocation size.

Enable warnings by default

Turn on -Wall and -pedantic by default if supported by the compiler.

Helper method for enum serialization

This is written in EMPIRE. I'll port it over and post the PR.

Merge kokkos into master

Once #1, #2, #3 are complete, the kokkos branch can be merged into master.

Clean up gcc warnings

Additional compile warnings that showed up using gcc on Linux.
make.log

Test n-dim code with gcc, icc compilers

Enable the new N-dimensional traversal code for serialization and equality comparison. Test if it works on Intel 18 compilers and gcc 5/6, etc.

Examples won't build on shiller

The line MyTest my_test_inst{10}; in serdes_example_1.cc and serdes_example_2.cc fails to compile on shiller due to the {10}.

test_object.cc calls init() method in check()

This looks to be wrong:

  void check() {
    EXPECT_EQ(x, x_val);
    EXPECT_EQ(y, y_val);
    EXPECT_EQ(vec.size(), 1UL);
    EXPECT_EQ(vec[0], vec_val);
    obj.init();   // <========== This here
    EXPECT_EQ(e8, e8_val);
    EXPECT_EQ(e64, e64_val);
    EXPECT_EQ(e0, e0_val);
  }

I think that should be obj.check()

Problem with serialization needed for EMPIRE

We have some data structures that are a view of view that we must serialize in EMPIRE. The code currently crashes for these:

Kokkos::View<Kokkos::View<double*>[3]> B_nodes_solution_;

Problems with Kokkos View trait: Kokkos::MemoryTraits<Kokkos::Atomic>

Since the PR Pr for gtest inclusion in checkpoint, vt-aut-build failed to compile from scratch at the checkpoint compilation step :
[ 37%] Building CXX object tests/CMakeFiles/test_kokkos_serialize_integration_mpi.dir/unit/main.cc.o clang: warning: argument unused during compilation: '--gcc-toolchain=/usr' /Users/perrinel/Dev/vt-auto-build/checkpoint-gtest/checkpoint/checkpoint/tests/unit/main.cc:4:10: fatal error: 'gtest/gtest.h' file not found #include <gtest/gtest.h> ^ /Users/perrinel/Dev/vt-auto-build/checkpoint-gtest/checkpoint/checkpoint/tests/unit/main.cc:4:10: fatal error: 'gtest/gtest.h' file not found #include <gtest/gtest.h> ^ /Users/perrinel/Dev/vt-auto-build/checkpoint-gtest/checkpoint/checkpoint/tests/unit/main.cc:4:10: fatal error: 'gtest/gtest.h' file not found #include <gtest/gtest.h> ^
In my case, I need to reput the

  target_link_libraries(${target} PRIVATE GTest::GTest)
  target_link_libraries(${target} PRIVATE GTest::Main)

at the checkpoint/tests/CMakeLists.txt to make that work again.
Any idea of the problem ? @nlslatt @lifflander

Compile warnings

Remove warnings from tests/unit/test_kokkos_serialize_integration.cc.

Write serializer for KokkosCrsMatrix

Fix new warning introduced by std::vector<bool> serializer overload

In file included from /home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/serdes_headers.h:58,
                 from /home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/serialization_library_headers.h:48,
                 from /home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/tests/unit/test_vector_bool_serialize.cc:47:
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/container/vector_serialize.h: In instantiation of ‘void serdes::serialize(Serializer&, std::vector<bool, VectorAllocator>&) [with Serializer = serdes::Sizer; VectorAllocator = std::allocator<bool>]’:
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:238:16:   required from ‘void serdes::SerializerDispatchNonByte<SerializerT, T>::apply(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U>*) [with U = std::vector<bool>; SerializerT = serdes::Sizer; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchNonByte<SerializerT, T>::hasNoninSerialize<U> = std::vector<bool>]’
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/dispatch/dispatch_serializer_nonbyte.h:161:29:   required from ‘void serdes::SerializerDispatchNonByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType) [with U = std::vector<bool>; SerializerT = serdes::Sizer; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int]’
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/dispatch/dispatch_serializer_byte.h:108:13:   required from ‘void serdes::SerializerDispatchByte<SerializerT, T>::operator()(SerializerT&, T*, serdes::SerialSizeType, serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U>*) [with U = std::vector<bool>; SerializerT = serdes::Sizer; T = std::vector<bool>; serdes::SerialSizeType = long unsigned int; serdes::SerializerDispatchByte<SerializerT, T>::isNotByteCopyType<U> = std::vector<bool>]’
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/dispatch/dispatch.impl.h:61:5:   required from ‘static serdes::SerialSizeType serdes::Dispatch<T>::sizeType(T&) [with T = std::vector<bool>; serdes::SerialSizeType = long unsigned int]’
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/dispatch/dispatch.impl.h:217:46:   required from ‘serdes::SerializedReturnType serdes::serializeType(T&, serdes::BufferObtainFnType) [with T = std::vector<bool>; serdes::SerializedReturnType = std::tuple<std::unique_ptr<serdes::Buffer, std::default_delete<serdes::Buffer> >, long unsigned int>; serdes::BufferObtainFnType = std::function<char*(long unsigned int)>]’
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/serialize_interface.impl.h:59:40:   required from ‘serialization::interface::SerializedReturnType serialization::interface::serialize(T&, serialization::interface::BufferCallbackType) [with T = std::vector<bool>; serialization::interface::SerializedReturnType = std::unique_ptr<serialization::interface::SerializedInfo>; serialization::interface::BufferCallbackType = std::function<char*(long unsigned int)>]’
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/tests/unit/test_vector_bool_serialize.cc:54:81:   required from here
/home/jliffla/vt-auto-build/vt-script-build-1/checkpoint/checkpoint/src/container/vector_serialize.h:77:24: warning: comparison of integer expressions of different signedness: ‘int’ and ‘std::vector<bool>::size_type’ {aka ‘long unsigned int’} [-Wsign-compare]
     for (auto i = 0; i < vec.size(); ++i) {

Unit test with MPI and serialization

Test serialization in the context of MPI for a more rigorous test (breaks pointers, etc)

0-dimensional serialization fails (Kokkos::View<double>)

Split kokkos tests into multiple files

Extract kokkos-serialize.cc into multiple files. Header file for common code. Test for 1,2,3-D, dynamic, integration, etc. into separate files.

Refine warning suppression

Push and pop diagnostics when suppressing warnings about unknown pragmas.

Change the way the checkpoint build system finds detector

Allow checkpoint to find detector when the two are dropped into a shared build system (i.e., when they are integrated into another build system as TPLs). I think this means it needs an alternative approach to using find_package_local. If this can already be done without modifications, please enlighten me as to how.

@PhilMiller @lifflander

Allow object reconstruction in place from user passed buffer

Add `package_ENABLE` as a boolean corresponding to `package_DIR`

Change macro optional_pkg_directory to create a flag and use it. Check for the case where package_ENABLE is true but package_DIR is empty.

Tests fail due to OpenMP warnings

Some checkpoint tests fail due to OpenMP warnings:

The following tests FAILED:
10 - checkpoint:test_kokkos_serialize_0d (Failed)
11 - checkpoint:test_kokkos_serialize_1d (Failed)
12 - checkpoint:test_kokkos_serialize_2d (Failed)
13 - checkpoint:test_kokkos_serialize_3d (Failed)
14 - checkpoint:test_kokkos_serialize_integration (Failed)
19 - checkpoint:test_kokkos_serialize_0d_mpi (Failed)
20 - checkpoint:test_kokkos_serialize_1d_mpi (Failed)
21 - checkpoint:test_kokkos_serialize_2d_mpi (Failed)
22 - checkpoint:test_kokkos_serialize_3d_mpi (Failed)
23 - checkpoint:test_kokkos_serialize_integration_mpi (Failed)

Kokkos::OpenMP::initialize WARNING: OMP_PROC_BIND environment variable not set
  In general, for best performance with OpenMP 4.0 or better set OMP_PROC_BIND=spread and OMP_PLACES=threads
  For best performance with OpenMP 3.1 set OMP_PROC_BIND=true
  For unit testing set OMP_PROC_BIND=false

Was the fail regex on the test intended to catch these particular warnings? If not, please revise the regex so this doesn't cause test failure.

LastTest.log

nvcc bug with typeid(std::tuple<int,int[6]>).name

Solution to work around is to reverse std::tuple order because the bug is not fixed until nvcc 10.1

Minimal reproducer:

#include <tuple>

void nvcc_bug_9_2() {
  typeid(std::tuple<char,int[3]>).name();
}

Output:

==================================================

# nvcc --version
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2018 NVIDIA Corporation
Built on Tue_Jun_12_23:07:04_CDT_2018
Cuda compilation tools, release 9.2, V9.2.148

==================================================


/net/sherlock/storage/fast/projects/sems/install/rhel7-x86_64/sems/compiler/gcc/7.2.0/base/include/c++/7.2.0/tuple(133): error: invalid initializer for array member "std::_Head_base<_Idx, _Head, false>::_M_head_impl [with _Idx=1UL, _Head=int [3]]"
          detected during:
            instantiation of "std::_Head_base<_Idx, _Head, false>::_Head_base(_UHead &&) [with _Idx=1UL, _Head=int [3], _UHead=int [3]]"
(373): here
            instantiation of "std::_Tuple_impl<_Idx, _Head>::_Tuple_impl(std::_Tuple_impl<_Idx, _Head> &&) [with _Idx=1UL, _Head=int [3]]"
/net/sherlock/storage/fast/projects/sems/install/rhel7-x86_64/sems/compiler/gcc/7.2.0/base/include/c++/7.2.0/type_traits(1207): here
            instantiation of class "std::__is_nt_constructible_impl<_Tp, _Arg> [with _Tp=std::_Tuple_impl<1UL, int [3]>, _Arg=std::_Tuple_impl<1UL, int [3]> &&]"
/net/sherlock/storage/fast/projects/sems/install/rhel7-x86_64/sems/compiler/gcc/7.2.0/base/include/c++/7.2.0/type_traits(144): here
            instantiation of class "std::__and_<_B1, _B2> [with _B1=std::is_constructible<std::_Tuple_impl<1UL, int [3]>, std::_Tuple_impl<1UL, int [3]> &&>, _B2=std::__is_nt_constructible_impl<std::_Tuple_impl<1UL, int [3]>, std::_Tuple_impl<1UL, int [3]> &&>]"
/net/sherlock/storage/fast/projects/sems/install/rhel7-x86_64/sems/compiler/gcc/7.2.0/base/include/c++/7.2.0/type_traits(1218): here
            instantiation of class "std::is_nothrow_constructible<_Tp, _Args...> [with _Tp=std::_Tuple_impl<1UL, int [3]>, _Args=<std::_Tuple_impl<1UL, int [3]> &&>]"
/net/sherlock/storage/fast/projects/sems/install/rhel7-x86_64/sems/compiler/gcc/7.2.0/base/include/c++/7.2.0/type_traits(1249): here
            instantiation of class "std::__is_nothrow_move_constructible_impl<_Tp, true> [with _Tp=std::_Tuple_impl<1UL, int [3]>]"
/net/sherlock/storage/fast/projects/sems/install/rhel7-x86_64/sems/compiler/gcc/7.2.0/base/include/c++/7.2.0/type_traits(1255): here
            instantiation of class "std::is_nothrow_move_constructible<_Tp> [with _Tp=std::_Tuple_impl<1UL, int [3]>]"
/net/sherlock/storage/fast/projects/sems/install/rhel7-x86_64/sems/compiler/gcc/7.2.0/base/include/c++/7.2.0/type_traits(144): here
            instantiation of class "std::__and_<_B1, _B2> [with _B1=std::is_nothrow_move_constructible<char>, _B2=std::is_nothrow_move_constructible<std::_Tuple_impl<1UL, int [3]>>]"
(225): here
            instantiation of "std::_Tuple_impl<_Idx, _Head, _Tail...>::_Tuple_impl(std::_Tuple_impl<_Idx, _Head, _Tail...> &&) [with _Idx=0UL, _Head=char, _Tail=<int [3]>]"
/home/jliffla/EMPIRE/TPL/checkpoint/tests/unit/test_kokkos_serialize_1d.cc(53): here

1 error detected in the compilation of "/tmp/tmpxft_0006bed6_00000000-6_test_kokkos_serialize_1d.cpp1.ii".
make[3]: *** [TPL/checkpoint/tests/CMakeFiles/test_kokkos_serialize_1d.dir/unit/test_kokkos_serialize_1d.cc.o] Error 1

Add example checkpointing bytes to file

Write a small example per request here:

https://cee-gitlab.sandia.gov/EM-Plasma/EMPIRE-PIC/issues/719

Serialize kokkos views

This is being implemented by @mperrinel with help from @nmm0

Please track progress on this in this issue.

Add direct size method to interface

Code coverage for Kokkos header files

Extract code coverage info for kokkos serialization code to better understand how comprehensive the serialization code is.

Add checkpoint label to ctest output

Instead of having the ctest output use the executable name as the test name, prepend the label "checkpoint" to it. For example, use "checkpoint:serdes_example_1" instead of simply "serdes_example_1" so that the source of the test is known when checkpoint is being built as a TPL.

Fix vector of bool (add correct overload)

std::vector<bool> has special semantics. A general serializer, as written in checkpoint, can't be used to serialize that vector because you can't take a reference to a single element (i.e., bit iterator __bit_iterator). Read this article: Vector of Bool

We need to write an overload for vector of bool in checkpoint so we can correctly serialize them. Add a test that exposes this problem in the checkpoint testing so it doesn't come up again.

Skips finding Kokkos when added as subdirectory of VT

It sets Kokkos_DIR_FOUND, but hasn't actually found it reeking havoc during compilation.

Implement non-intrusive reconstruct method

Clean up compile warnings

Clean up all compile warnings so that checkpoint builds cleanly.

checkpoint/src/container/view_traverse_manual.h:81:11: warning: unused type alias 'BaseType' [-Wunused-local-typedef]
using BaseType = typename CountDims::BaseT;

checkpoint/src/container/view_traverse_manual.h:82:11: warning: unused type alias 'MemoryTraitsType' [-Wunused-local-typedef]
using MemoryTraitsType = typename ViewType::traits::memory_traits;

checkpoint/src/container/view_serialize.h:193:9: warning: unused type alias 'ViewTraitsType' [-Wunused-local-typedef]
using ViewTraitsType = Kokkos::ViewTraits<T,Args...>;

checkpoint/src/serdes_common.h:14:37: warning: named variadic macros are a GNU extension [-Wvariadic-macros]
#define debug_serdes(debug_str, args...)

checkpoint/src/container/view_serialize.h:43:39: warning: named variadic macros are a GNU extension [-Wvariadic-macros]
#define DEBUG_PRINT_SERDES(str, args...)

Remove gmock

Serializing virtual hierarchies of classes

Add cmake option for enabling Kokkos serialization

Kokkos::View serialization does not work for const types

None of the tests exercise the ability to serialize T, where T = const T'

For example, try running the serializer on: Kokkos::View<const signed char**>. This is actually one of the types in EMPIRE.

The operator accesses the data as if it can set it. We need an overload that handles the case where it cannot be set. Either find Kokkos functionality for doing this, or regress to the contiguous code which does not call operator()(..), accessing the data directly through the pointer. Code will not currently compile so that might be the best solution in the short term.

checkpoint/examples/serdes_example_nonintrusive.cc:36:32: warning: format
      specifies type 'void *' but the argument has type 'SerialByteType *' (aka 'char *') [-Wformat-pedantic]
  printf("ptr=%p, size=%ld\n", buf->getBuffer(), buf_size);
              ~~               ^~~~~~~~~~~~~~~~
              %s

checkpoint/examples/../src/dispatch/dispatch.impl.h:170:15: warning: unused
      variable 'buf' [-Wunused-variable]
  auto const& buf = managed->getBuffer();

checkpoint/src/container/view_serialize.h:196:31: warning: unused variable 'is_managed'
      [-Wunused-variable]
  static constexpr auto const is_managed = ViewType::traits::is_managed;

checkpoint/tests/unit/test_reconstruct.cc:19:28: warning: unused variable 'x_val'
      [-Wunused-const-variable]
static constexpr int const x_val = 29;

checkpoint/tests/unit/test_reconstruct.cc:20:28: warning: unused variable 'y_val'
      [-Wunused-const-variable]
static constexpr int const y_val = 31;

checkpoint/tests/unit/test_reconstruct.cc:21:28: warning: unused variable 'z_val'
      [-Wunused-const-variable]
static constexpr int const z_val = 37;

checkpoint/tests/unit/test_reconstruct.cc:23:28: warning: unused variable 'vec_val'
      [-Wunused-const-variable]
static constexpr int const vec_val = 41;

checkpoint/tests/unit/test_bytecopy_trait.cc:39:9: warning: unused variable 'dest' [-Wunused-variable]
  auto& dest = *tptr;

checkpoint/tests/unit/tests_mpi/mpi-init.h:35:9: warning: unused variable 'mpiError' [-Wunused-variable]
    int mpiError = MPI_Finalize();

checkpoint/tests/unit/test_kokkos_integration_commons.h:36:24: warning: field 'val2' will be initialized
      after field 'vec' [-Wreorder]
    : val1(gold_val1), val2(gold_val2), vec({gold_val3,gold_val3+1,gold_val3+2})

checkpoint/tests/unit/test_kokkos_integration_commons.h:36:24: warning: field 'val2' will be initialized
      after field 'vec' [-Wreorder]
    : val1(gold_val1), val2(gold_val2), vec({gold_val3,gold_val3+1,gold_val3+2})

checkpoint/tests/unit/test_kokkos_serialize_integration.cc:39:8: warning: unused variable 'out'
      [-Wunused-variable]
  auto out = deserialize<ViewType>(std::move(ret));

unit tests not being disabled appropriately

gtest was assumed available in tpl mode.