Merge pull request #12 from AmineKhaldi/update_upstream

Update to latest madmax with k33 and k34 support

.github/workflows/build-arm64.yml

@@ -28,7 +28,9 @@ jobs:
       uses: actions/[email protected]
       with:
         name: Debug build
-        path: ${{github.workspace}}/build/chia_plot
+        path: |
+          ${{github.workspace}}/build/chia_plot
+          ${{github.workspace}}/build/chia_plot_k34
 
   build-release:
     runs-on: [ARM64]
@@ -46,7 +48,9 @@ jobs:
       uses: actions/[email protected]
       with:
         name: Release build
-        path: ${{github.workspace}}/build/chia_plot
+        path: |
+          ${{github.workspace}}/build/chia_plot
+          ${{github.workspace}}/build/chia_plot_k34
 
     - name: Get tag name
       if: startsWith(github.ref, 'refs/tags/')

.github/workflows/build-linux.yml

@@ -28,7 +28,9 @@ jobs:
       uses: actions/[email protected]
       with:
         name: Debug build
-        path: ${{github.workspace}}/build/chia_plot
+        path: |
+          ${{github.workspace}}/build/chia_plot
+          ${{github.workspace}}/build/chia_plot_k34
 
   build-release:
     runs-on: ubuntu-latest
@@ -46,7 +48,9 @@ jobs:
       uses: actions/[email protected]
       with:
         name: Release build
-        path: ${{github.workspace}}/build/chia_plot
+        path: |
+          ${{github.workspace}}/build/chia_plot
+          ${{github.workspace}}/build/chia_plot_k34
 
     - name: Get tag name
       if: startsWith(github.ref, 'refs/tags/')

.github/workflows/build-m1.yml

@@ -31,7 +31,9 @@ jobs:
       uses: actions/[email protected]
       with:
         name: Debug build
-        path: ${{github.workspace}}/build/chia_plot
+        path: |
+          ${{github.workspace}}/build/chia_plot
+          ${{github.workspace}}/build/chia_plot_k34
 
   build-release:
     runs-on: [m1]
@@ -52,7 +54,9 @@ jobs:
       uses: actions/[email protected]
       with:
         name: Release build
-        path: ${{github.workspace}}/build/chia_plot
+        path: |
+          ${{github.workspace}}/build/chia_plot
+          ${{github.workspace}}/build/chia_plot_k34
 
     - name: Get tag name
       if: startsWith(github.ref, 'refs/tags/')

.github/workflows/build-osx.yml

@@ -31,7 +31,9 @@ jobs:
       uses: actions/[email protected]
       with:
         name: Debug build
-        path: ${{github.workspace}}/build/chia_plot
+        path: |
+          ${{github.workspace}}/build/chia_plot
+          ${{github.workspace}}/build/chia_plot_k34
 
   build-release:
     runs-on: macOS-latest
@@ -52,7 +54,9 @@ jobs:
       uses: actions/[email protected]
       with:
         name: Release build
-        path: ${{github.workspace}}/build/chia_plot
+        path: |
+          ${{github.workspace}}/build/chia_plot
+          ${{github.workspace}}/build/chia_plot_k34
 
     - name: Get tag name
       if: startsWith(github.ref, 'refs/tags/')

.github/workflows/build-windows.yml

@@ -37,7 +37,9 @@ jobs:
         uses: actions/[email protected]
         with:
           name: MSVC Release build
-          path: ${{github.workspace}}/build/chia_plot.exe
+          path: |
+            ${{github.workspace}}/build/chia_plot.exe
+            ${{github.workspace}}/build/chia_plot_k34.exe
 
   upload-release-artifacts:
     runs-on: ubuntu-latest

CMakeLists.txt

@@ -161,6 +161,24 @@ target_link_libraries(chia_plot
     bls
 )
 
+add_executable(chia_plot_k34 src/chia_plot.cpp)
+
+target_include_directories(chia_plot_k34
+  PRIVATE
+    ${libbech32_SOURCE_DIR}/include/libbech32
+)
+
+target_compile_definitions(chia_plot_k34
+  PUBLIC
+    CHIA_K34=1
+)
+
+target_link_libraries(chia_plot_k34
+  PRIVATE
+    chia_plotter
+    bls
+)
+
 if(CHIA_PLOTTER_BUILD_TESTS)
   add_executable(test_copy test/test_copy.cpp)
   target_link_libraries(test_copy PRIVATE chia_plotter)

README.md

@@ -36,6 +36,7 @@ Usage:
   -c, --contract arg   Pool Contract Address (62 chars)
   -f, --farmerkey arg  Farmer Public Key (48 bytes)
   -G, --tmptoggle      Alternate tmpdir/tmpdir2 (default = false)
+  -D, --directout      Create plot directly in finaldir (default = false)
   -K, --rmulti2 arg    Thread multiplier for P2 (default = 1)
       --help           Print help
 ```

include/chia/DiskSort.hpp

@@ -24,7 +24,7 @@ void DiskSort<T, Key>::bucket_t::open(const char* mode)
 	}
 	file = FOPEN(file_name.c_str(), mode);
 	if(!file) {
-		throw std::runtime_error("fopen() failed");
+		throw std::runtime_error("fopen() failed with: " + std::string(std::strerror(errno)));
 	}
 }
 
@@ -34,7 +34,7 @@ void DiskSort<T, Key>::bucket_t::write(const void* data, size_t count)
 	std::lock_guard<std::mutex> lock(mutex);
 	if(file) {
 		if(fwrite(data, T::disk_size, count, file) != count) {
-			throw std::runtime_error("fwrite() failed");
+			throw std::runtime_error("fwrite() failed with: " + std::string(std::strerror(errno)));
 		}
 		num_entries += count;
 	}
@@ -44,7 +44,9 @@ template<typename T, typename Key>
 void DiskSort<T, Key>::bucket_t::close()
 {
 	if(file) {
-		fclose(file);
+		if(fclose(file)) {
+			throw std::runtime_error("fclose() failed with: " + std::string(std::strerror(errno)));
+		}
 		file = nullptr;
 	}
 }
@@ -67,7 +69,8 @@ void DiskSort<T, Key>::WriteCache::add(const T& entry)
 {
 	const size_t index = Key{}(entry) >> key_shift;
 	if(index >= buckets.size()) {
-		throw std::logic_error("bucket index out of range");
+		throw std::logic_error("bucket index out of range: "
+				+ std::to_string(index) + " >= " + std::to_string(buckets.size()));
 	}
 	auto& buffer = buckets[index];
 	if(buffer.count >= buffer.capacity) {
@@ -197,7 +200,7 @@ void DiskSort<T, Key>::read_bucket(	std::pair<size_t, size_t>& index,
 	{
 		const size_t num_entries = std::min(buffer.capacity, bucket.num_entries - i);
 		if(fread(buffer.data, T::disk_size, num_entries, bucket.file) != num_entries) {
-			throw std::runtime_error("fread() failed");
+			throw std::runtime_error("fread() failed with: " + std::string(std::strerror(errno)));
 		}
 		for(size_t k = 0; k < num_entries; ++k) {
 			T entry;

include/chia/DiskTable.h

@@ -36,6 +36,9 @@ class DiskTable {
 	{
 		if(!num_entries) {
 			file_out = FOPEN(file_name.c_str(), "wb");
+			if(!file_out) {
+				throw std::runtime_error("fopen() failed with: " + std::string(std::strerror(errno)));
+			}
 		}
 	}
 
@@ -71,7 +74,7 @@ class DiskTable {
 		{
 			FILE* file = FOPEN(file_name.c_str(), "rb");
 			if(!file) {
-				throw std::runtime_error("fopen() failed");
+				throw std::runtime_error("fopen() failed with: " + std::string(std::strerror(errno)));
 			}
 			auto& local = pool.get_local(i);
 			local.file = file;
@@ -101,8 +104,11 @@ class DiskTable {
 	}
 
 	void flush() {
+		if(!file_out) {
+			throw std::logic_error("read only");
+		}
 		if(fwrite(cache.data, cache.entry_size, cache.count, file_out) != cache.count) {
-			throw std::runtime_error("fwrite() failed");
+			throw std::runtime_error("fwrite() failed with: " + std::string(std::strerror(errno)));
 		}
 		num_entries += cache.count;
 		cache.count = 0;
@@ -122,10 +128,10 @@ class DiskTable {
 					local_t& local) const
 	{
 		if(int err = FSEEK(local.file, param.first * T::disk_size, SEEK_SET)) {
-			throw std::runtime_error("fseek() failed");
+			throw std::runtime_error("fseek() failed with: " + std::string(std::strerror(errno)));
 		}
 		if(fread(local.buffer, T::disk_size, param.second, local.file) != param.second) {
-			throw std::runtime_error("fread() failed");
+			throw std::runtime_error("fread() failed with: " + std::string(std::strerror(errno)));
 		}
 		auto& entries = out.first;
 		entries.resize(param.second);

include/chia/bits.hpp

@@ -554,4 +554,61 @@ using Bits = BitsGeneric<SmallVector>;
 using ParkBits = BitsGeneric<ParkVector>;
 using LargeBits = BitsGeneric<LargeVector>;
 
+
+inline
+int write_bits(uint64_t* dst, const uint64_t value, const int bit_offset, const int num_bits)
+{
+	assert(num_bits <= 64);
+	const int free_bits = 64 - (bit_offset % 64);
+	if(free_bits >= num_bits) {
+		dst[bit_offset / 64]     |= bswap_64(value << (free_bits - num_bits));
+	} else {
+		const int suffix_size = num_bits - free_bits;
+		const uint64_t suffix = value & ((uint64_t(1) << suffix_size) - 1);
+		dst[bit_offset / 64]     |= bswap_64(value >> suffix_size);			// prefix (high bits)
+		dst[bit_offset / 64 + 1] |= bswap_64(suffix << (64 - suffix_size));	// suffix (low bits)
+	}
+	return bit_offset + num_bits;
+}
+
+inline
+int append_bits(uint64_t* dst, const uint64_t* src, const int bit_offset, const int num_bits)
+{
+	int i = 0;
+	int offset = bit_offset;
+	int num_left = num_bits;
+	while(num_left > 0) {
+		int bits = 64;
+		uint64_t value = bswap_64(src[i]);
+		if(num_left < 64) {
+			bits = num_left;
+			value >>= (64 - num_left);
+		}
+		offset = write_bits(dst, value, offset, bits);
+		num_left -= bits;
+		i++;
+	}
+	return offset;
+}
+
+inline
+int slice_bits(uint64_t* dst, const uint64_t* src, const int bit_offset, const int num_bits)
+{
+	int count = 0;
+	int offset = bit_offset;
+	int num_left = num_bits;
+	while(num_left > 0) {
+		const int shift = offset % 64;
+		const int bits = std::min(num_left, 64 - shift);
+		uint64_t value = bswap_64(src[offset / 64]) << shift;
+		if(bits < 64) {
+			value >>= (64 - bits);
+		}
+		count = write_bits(dst, value, count, bits);
+		offset += bits;
+		num_left -= bits;
+	}
+	return count;
+}
+
 #endif  // SRC_CPP_BITS_HPP_

include/chia/copy.h

@@ -14,6 +14,8 @@
 
 #include <cstdio>
 #include <cstdint>
+#include <cstring>
+#include <errno.h>
 
 #ifdef _MSC_VER
 #include <filesystem>
@@ -27,21 +29,23 @@ uint64_t copy_file(const std::string& src_path, const std::string& dst_path)
 {
 	FILE* src = FOPEN(src_path.c_str(), "rb");
 	if(!src) {
-		throw std::runtime_error("fopen() failed for " + src_path);
+		throw std::runtime_error("fopen() failed for " + src_path + " (" + std::string(std::strerror(errno)) + ")");
 	}
 	FILE* dst = FOPEN(dst_path.c_str(), "wb");
 	if(!dst) {
+		const auto err = errno;
 		fclose(src);
-		throw std::runtime_error("fopen() failed for " + dst_path);
+		throw std::runtime_error("fopen() failed for " + dst_path + " (" + std::string(std::strerror(err)) + ")");
 	}
 	uint64_t total_bytes = 0;
 	std::vector<uint8_t> buffer(g_read_chunk_size * 16);
 	while(true) {
 		const auto num_bytes = fread(buffer.data(), 1, buffer.size(), src);
 		if(fwrite(buffer.data(), 1, num_bytes, dst) != num_bytes) {
+			const auto err = errno;
 			fclose(src);
 			fclose(dst);
-			throw std::runtime_error("fwrite() failed on " + dst_path);
+			throw std::runtime_error("fwrite() failed on " + dst_path + " (" + std::string(std::strerror(err)) + ")");
 		}
 		total_bytes += num_bytes;
 		if(num_bytes < buffer.size()) {
@@ -51,7 +55,7 @@ uint64_t copy_file(const std::string& src_path, const std::string& dst_path)
 	fclose(src);
 
 	if(fclose(dst)) {
-		throw std::runtime_error("fclose() failed on " + dst_path);
+		throw std::runtime_error("fclose() failed on " + dst_path + " (" + std::string(std::strerror(errno)) + ")");
 	}
 	return total_bytes;
 }

include/chia/entries.h

@@ -8,9 +8,26 @@
 #ifndef INCLUDE_CHIA_ENTRIES_H_
 #define INCLUDE_CHIA_ENTRIES_H_
 
+#include <chia/util.hpp>
+
 #include <cstdio>
 #include <cstdint>
 
+#ifdef CHIA_K34
+	constexpr int KMAX = 34;
+	constexpr int PMAX = 35;
+	constexpr int KBYTES = 5;
+	typedef uint64_t uintkx_t;
+	typedef uint128_t uintlp_t;
+#else
+#define CHIA_K32
+	constexpr int KMAX = 32;
+	constexpr int PMAX = 32;
+	constexpr int KBYTES = 4;
+	typedef uint32_t uintkx_t;
+	typedef uint64_t uintlp_t;
+#endif
+
 
 template<typename T>
 bool write_entry(FILE* file, const T& entry) {