TinyNPY.cpp

////////////////////////////////////////////////////////////////////
// TinyNPY.cpp
//
// Copyright 2007 cDc@seacave
// Distributed under the Boost Software License, Version 1.0
// (See http://www.boost.org/LICENSE_1_0.txt)

#include "TinyNPY.h"
#include <complex>
#include <functional>
#include <regex>
#include <sstream>
#include <zlib.h>


// D E F I N E S ///////////////////////////////////////////////////


// S T R U C T S ///////////////////////////////////////////////////

// Safely read little-endian uint16_t from buffer
inline uint16_t ReadLE16(const uint8_t* p) {
	return (uint16_t(p[1]) << 8) | uint16_t(p[0]);
}
// Safely read little-endian uint32_t from buffer
inline uint32_t ReadLE32(const uint8_t* p) {
	return (uint32_t(p[3]) << 24) | (uint32_t(p[2]) << 16) | 
	       (uint32_t(p[1]) << 8) | uint32_t(p[0]);
}
// Safely read little-endian uint64_t from buffer
inline uint64_t ReadLE64(const uint8_t* p) {
	return (uint64_t(p[7]) << 56) |
		   (uint64_t(p[6]) << 48) |
		   (uint64_t(p[5]) << 40) |
		   (uint64_t(p[4]) << 32) |
		   (uint64_t(p[3]) << 24) |
		   (uint64_t(p[2]) << 16) |
		   (uint64_t(p[1]) << 8)  |
		   uint64_t(p[0]);
}
/*----------------------------------------------------------------*/


// Invoke a function when the object is destroyed,
// typically at scope exit if the object is allocated on the stack
template <typename Functor=std::function<void()>>
class TScopeExitRun
{
public:
	TScopeExitRun(Functor f) : functor(f) {}
	~TScopeExitRun() { functor(); }
	void Reset(Functor f) { functor = f; }

protected:
	Functor functor;
};
typedef class TScopeExitRun<> ScopeExitRun;
/*----------------------------------------------------------------*/


// input
LPCSTR NpyArray::ParseHeaderNPY(const std::string& header, shape_t& shape, size_t& wordSize, char& type, bool& fortranOrder)
{
	ASSERT(header[header.size() - 1] == '\n');

	// fortran order
	size_t loc1 = header.find("fortran_order");
	if (loc1 == std::string::npos)
		return "error: failed to find header keyword 'fortran_order'";
	fortranOrder = (header.substr(loc1+16, 4) == "True");

	// shape
	loc1 = header.find("(");
	size_t loc2 = header.find(")");
	if (loc1 == std::string::npos || loc2 == std::string::npos)
		return "error: failed to find header keyword '(' or ')'";

	shape.clear();
	std::regex num_regex("[0-9][0-9]*");
	std::smatch sm;
	std::string strShape = header.substr(loc1 + 1, loc2 - loc1 - 1);
	while (std::regex_search(strShape, sm, num_regex)) {
		shape.push_back(std::stoi(sm[0].str()));
		strShape = sm.suffix().str();
	}
	// Empty shape () is valid - represents a 0-D scalar array with 1 value

	// endian, word size, data type
	// byte order code | stands for not applicable. 
	// not sure when this applies except for byte array
	loc1 = header.find("descr");
	if (loc1 == std::string::npos)
		return "error: failed to find header keyword 'descr'";
	loc1 += 9;
	const bool littleEndian = (header[loc1] == '<' || header[loc1] == '|');
	ASSERT(littleEndian);
	type = header[loc1+1];

	const std::string str_ws = header.substr(loc1+2);
	loc2 = str_ws.find("'");
	wordSize = std::stoi(str_ws.substr(0, loc2));
	if (wordSize == 0 || wordSize > 65536)
		return "error: invalid word size";
	// most data types like 'i', 'f', 'c' specifie word size in bytes
	// while U specifies characters in the header string, each 4 bytes,
	// so here we multiply by 4 to get the actual word size in bytes
	if (type == 'U')
		wordSize *= 4;
	return NULL;
}

LPCSTR NpyArray::ParseHeaderNPY(const uint8_t* buffer, shape_t& shape, size_t& wordSize, char& type, bool& fortranOrder)
{
	if (buffer[0] != (uint8_t)0x93 || _tcsncmp(reinterpret_cast<const char*>(buffer+1), "NUMPY", 5) != 0)
		return "error: invalid header id";
	// parse the length of the header data
	uint32_t lenHeader, offset;
	ASSERT(buffer[7] >= 0); // minor version number of the file format
	if (buffer[6] > 1) { // major version number of the file format
		// little-endian unsigned int
		lenHeader = ReadLE32(buffer+8);
		offset = 12;
	} else {
		// little-endian unsigned short int
		lenHeader = ReadLE16(buffer+8);
		offset = 10;
	}
	// Sanity check: limit header size to prevent DoS
	if (lenHeader > 1024 * 1024) // 1 MB should be plenty
		return "error: header size exceeds maximum allowed";
	const std::string header(reinterpret_cast<const char*>(buffer+offset), lenHeader);
	return ParseHeaderNPY(header, shape, wordSize, type, fortranOrder);
}

LPCSTR NpyArray::ParseHeaderNPY(FILE* fp, shape_t& shape, size_t& wordSize, char& type, bool& fortranOrder)
{
	char buffer[32];
	if (fread(buffer, sizeof(char), 10, fp) != 10 ||
		buffer[0] != (char)0x93 || _tcsncmp(buffer+1, "NUMPY", 5) != 0)
		return "error: invalid header id";
	// parse the length of the header data
	uint32_t lenHeader;
	ASSERT(buffer[7] >= 0); // minor version number of the file format
	if (buffer[6] > 1) { // major version number of the file format
		// little-endian unsigned int
		fread(buffer+10, sizeof(char), 2, fp);
		lenHeader = (uint32_t(buffer[11])<<24)|(uint32_t(buffer[10])<<16)|(uint32_t(buffer[9])<<8)|uint32_t(buffer[8]);
	} else {
		// little-endian unsigned short int
		lenHeader = (uint16_t(buffer[9])<<8)|uint16_t(buffer[8]);
	}
	std::string header(lenHeader, '\0');
	if (fread(&header[0], sizeof(char), lenHeader, fp) != lenHeader)
		return "error: invalid header";
	return ParseHeaderNPY(header, shape, wordSize, type, fortranOrder);
}

LPCSTR NpyArray::ParseFooterZIP(FILE* fp, uint16_t& nrecs, size_t& globalHeaderSize, size_t& globalHeaderOffset)
{
	char footer[32];
	fseek(fp, -22, SEEK_END);
	if (fread(footer, sizeof(char), 22, fp) != 22)
		return "error: failed footer";
	// Use safe little-endian reads to avoid alignment issues
	const uint16_t diskNo = ReadLE16((uint8_t*)footer+4); ASSERT(diskNo == 0);
	const uint16_t diskStart = ReadLE16((uint8_t*)footer+6); ASSERT(diskStart == 0);
	const uint16_t nrecsOnDisk = ReadLE16((uint8_t*)footer+8);
	nrecs = ReadLE16((uint8_t*)footer+10); ASSERT(nrecsOnDisk == nrecs);
	globalHeaderSize = ReadLE32((uint8_t*)footer+12);
	globalHeaderOffset = ReadLE32((uint8_t*)footer+16);
	const uint16_t lenComment = ReadLE16((uint8_t*)footer+20); ASSERT(lenComment == 0);
	return NULL;
}

LPCSTR NpyArray::LoadNPY(FILE* fp)
{
	Release();
	LPCSTR ret = ParseHeaderNPY(fp, shape, wordSize, type, fortranOrder);
	if (ret != NULL)
		return ret;
	Initialize();
	const size_t nread = fread(Data(), 1, SizeBytes(), fp);
	if (nread != SizeBytes())
		return "error: failed fread";
	return NULL;
}

LPCSTR NpyArray::LoadNPY(std::string filename)
{
	FILE* fp = fopen(filename.c_str(), "rb");
	if (!fp)
		return "error: unable to open file";
	const ScopeExitRun closeFp([&]() { fclose(fp); });
	return LoadNPY(fp);
}

LPCSTR NpyArray::LoadNPZ(FILE* fp, uint32_t comprBytes, uint32_t uncomprBytes)
{
	// Validate uncompressed size is reasonable and at least as large as needed
	if (uncomprBytes == 0 || uncomprBytes > 1024*1024*1024) // 1 GB limit
		return "error: invalid uncompressed size";

	constexpr uint32_t MAX_HEADER_SIZE = 1024 * 1024; // keep header bounded
	constexpr size_t kInChunk = 64 * 1024; // 64 KB chunks for streaming
	std::vector<uint8_t> inBuf(kInChunk);
	std::vector<uint8_t> outBuf(kInChunk);
	std::vector<uint8_t> headerBuf;
	headerBuf.reserve(1024);

	z_stream d_stream;
	d_stream.zalloc = Z_NULL;
	d_stream.zfree = Z_NULL;
	d_stream.opaque = Z_NULL;
	d_stream.avail_in = 0;
	d_stream.next_in = Z_NULL;
	int err = inflateInit2(&d_stream, -MAX_WBITS);
	if (err != Z_OK)
		return "error: can not init inflate";

	bool headerDone = false;
	size_t headerExpected = 0; // total header bytes (preamble + dict)
	size_t dataWritten = 0;
	size_t dataTotal = 0;

	bool headerTooLarge = false;
	auto decodeHeaderSize = [&](const std::vector<uint8_t>& buf) -> bool {
		// Need at least magic(6) + ver(2) + len (2 or 4)
		if (buf.size() < 10)
			return false;
		const uint8_t ver = buf[6];
		const size_t offset = (ver > 1) ? 12 : 10;
		if (buf.size() < offset)
			return false;
		uint32_t lenHeader = (ver > 1) ? ReadLE32(buf.data()+8) : ReadLE16(buf.data()+8);
		if (lenHeader > MAX_HEADER_SIZE) {
			headerTooLarge = true;
			return false; // too large
		}
		headerExpected = offset + lenHeader;
		return true;
	};

	while (comprBytes > 0) {
		const size_t toRead = std::min(kInChunk, static_cast<size_t>(comprBytes));
		if (fread(inBuf.data(), 1, toRead, fp) != toRead) {
			inflateEnd(&d_stream);
			return "error: failed fread";
		}
		comprBytes -= static_cast<uint32_t>(toRead);

		d_stream.next_in = inBuf.data();
		d_stream.avail_in = static_cast<uInt>(toRead);

		while (d_stream.avail_in > 0) {
			d_stream.next_out = outBuf.data();
			d_stream.avail_out = static_cast<uInt>(outBuf.size());
			const int zret = inflate(&d_stream, Z_NO_FLUSH);
			if (zret != Z_OK && zret != Z_STREAM_END) {
				inflateEnd(&d_stream);
				return "error: can not uncompress";
			}

			const size_t produced = outBuf.size() - d_stream.avail_out;
			size_t idx = 0;

			while (idx < produced) {
				if (!headerDone) {
					// Still collecting header bytes
					headerBuf.insert(headerBuf.end(), outBuf.begin()+idx, outBuf.begin()+produced);
					idx = produced; // consumed all produced bytes into headerBuf

					// If we don't yet know header size, try to decode
					if (headerExpected == 0) {
						if (!decodeHeaderSize(headerBuf)) {
							if (headerTooLarge || headerBuf.size() > MAX_HEADER_SIZE) {
								inflateEnd(&d_stream);
								return "error: header size exceeds maximum allowed";
							}
							continue; // need more bytes
						}
					}

					// If we have full header, parse it
					if (headerExpected && headerBuf.size() >= headerExpected) {
						LPCSTR ret = ParseHeaderNPY(headerBuf.data(), shape, wordSize, type, fortranOrder);
						if (ret != NULL) {
							inflateEnd(&d_stream);
							return ret;
						}
						Initialize();
						dataTotal = SizeBytes();
						// Copy any data that already came with the header buffer
						const size_t tail = headerBuf.size() - headerExpected;
						if (tail) {
							const size_t toCopy = std::min(tail, dataTotal);
							memcpy(Data(), headerBuf.data()+headerExpected, toCopy);
							dataWritten += toCopy;
						}
						headerDone = true;
					}
				} else {
					// Header is done; stream directly into destination buffer
					const size_t remaining = produced - idx;
					const size_t space = dataTotal - dataWritten;
					const size_t toCopy = std::min(remaining, space);
					memcpy(Data()+dataWritten, outBuf.data()+idx, toCopy);
					dataWritten += toCopy;
					idx += toCopy;
					if (toCopy < remaining) {
						inflateEnd(&d_stream);
						return "error: unexpected extra data";
					}
				}
			}
			if (zret == Z_STREAM_END)
				break;
		}
	}

	inflateEnd(&d_stream);

	if (!headerDone)
		return "error: incomplete header";
	if (dataWritten != dataTotal)
		return "error: truncated data";
	return NULL;
}

LPCSTR NpyArray::LoadNPZ(std::string filename, std::string varname)
{
	Release();
	FILE* fp = fopen(filename.c_str(), "rb");
	if (!fp)
		return "error: unable to open file";
	const ScopeExitRun closeFp([&]() { fclose(fp); });
	while (true) {
		const LPCSTR ret = LoadArrayNPZ(fp, varname, *this);
		if (ret == NULL && !IsEmpty())
			return NULL;
		if (ret == (const char*)1)
			break;
	}
	return "error: variable name not found";
}

LPCSTR NpyArray::LoadNPZ(std::string filename, npz_t& arrays)
{
	FILE* fp = fopen(filename.c_str(), "rb");
	if (!fp)
		return "error: unable to open file";
	const ScopeExitRun closeFp([&]() { fclose(fp); });
	while (true) {
		NpyArray arr;
		std::string varname;
		const LPCSTR ret = LoadArrayNPZ(fp, varname, arr);
		if (ret == (const char*)1)
			break;
		if (ret != NULL)
			return ret;
		arrays.emplace(varname, std::move(arr));
	}
	return NULL;
}

LPCSTR NpyArray::LoadArrayNPZ(FILE* fp, std::string& varname, NpyArray& arr)
{
	uint8_t localHeader[32];
	if (fread(localHeader, sizeof(uint8_t), 30, fp) != 30)
		return "error: failed fread";

	// if we've reached the global header, stop reading
	if (localHeader[2] != 0x03 || localHeader[3] != 0x04)
		return (const char*)1;

	// read in the variable name
	const uint16_t lenName = ReadLE16(localHeader+26);
	std::string vname(lenName, ' ');
	if (fread(&vname[0], sizeof(char), lenName, fp) != lenName)
		return "error: failed fread";

	// erase the lagging .npy        
	vname.erase(vname.end()-4, vname.end());

	// read in the extra field
	const uint16_t lenExtraField = ReadLE16(localHeader+28);
	std::vector<uint8_t> extraBuf(lenExtraField);
	if (lenExtraField > 0) {
		if (fread(extraBuf.data(), 1, lenExtraField, fp) != lenExtraField)
			return "error: failed fread extra field";
	}

	// Extract compression method and initial size values
	const uint16_t comprMethod = ReadLE16(localHeader+8);
	uint32_t comprBytes = ReadLE32(localHeader+18);
	uint32_t uncomprBytes = ReadLE32(localHeader+22);

	// Handle ZIP64: if sizes are 0xFFFFFFFF, read from extra field
	if (comprBytes == 0xFFFFFFFF || uncomprBytes == 0xFFFFFFFF) {
		// Parse ZIP64 extra field (header ID 0x0001)
		bool found_zip64 = false;
		for (size_t i = 0; i + 4 <= lenExtraField; ) {
			uint16_t headerId = ReadLE16(&extraBuf[i]);
			uint16_t dataSize = ReadLE16(&extraBuf[i+2]);
			if (headerId == 0x0001) {
				// ZIP64 extra field found
				if (i + 4 + dataSize > lenExtraField || dataSize < 16)
					return "error: invalid ZIP64 extra field";
				// Read 64-bit sizes (uncompressed first, then compressed)
				if (uncomprBytes == 0xFFFFFFFF && i + 12 <= lenExtraField)
					uncomprBytes = (uint32_t)ReadLE64(extraBuf.data()+i+4);
				if (comprBytes == 0xFFFFFFFF && i + 20 <= lenExtraField)
					comprBytes = (uint32_t)ReadLE64(extraBuf.data()+i+12);
				found_zip64 = true;
				break;
			}
			i += 4 + dataSize;
		}
		if (!found_zip64)
			return "error: ZIP64 sizes but no ZIP64 extra field";
	}

	if (varname.empty() || varname == vname) {
		// read current array
		if (varname.empty())
			varname = vname;
		if (comprMethod == 0)
			return arr.LoadNPY(fp);
		if (comprBytes == 0 || uncomprBytes == 0)
			return "error: invalid compression sizes";
		return arr.LoadNPZ(fp, comprBytes, uncomprBytes);
	}

	// skip current array data
	const uint32_t size = comprMethod == 0 ? uncomprBytes : comprBytes;
	fseek(fp, size, SEEK_CUR);
	return NULL;
}
/*----------------------------------------------------------------*/



// output
std::vector<char> NpyArray::CreateHeaderNPY(const shape_t& shape, char type, size_t wordSize)
{
	std::vector<char> dict;
	Add(dict, "{'descr': '");
	#if __BYTE_ORDER == __LITTLE_ENDIAN
	Add(dict, '<');
	#else
	Add(dict, '>');
	#endif
	Add(dict, type);
	Add(dict, std::to_string(type == 'U' ? wordSize / 4 : wordSize));
	Add(dict, "', 'fortran_order': False, 'shape': (");
	// Empty shape represents a 0-D scalar array
	if (!shape.empty()) {
		Add(dict, std::to_string(shape[0]));
		for (size_t i = 1; i < shape.size(); i++) {
			Add(dict, ", ");
			Add(dict, std::to_string(shape[i]));
		}
	}
	Add(dict, "), }");
	// pad with spaces so that preamble+dict is modulo 16 bytes
	// preamble is 10/12 bytes and dict needs to end with \n
	char ver = 1;
	size_t remainder = 16 - (10 + dict.size()) % 16;
	if (dict.size() + remainder > 65535) {
		ver = 2;
		remainder = 16 - (12 + dict.size()) % 16;
	}
	dict.insert(dict.end(), remainder, ' ');
	dict.back() = '\n';

	std::vector<char> header;
	Add(header, (char)0x93);
	Add(header, "NUMPY");
	Add(header, ver); // major version of numpy format
	Add(header, (char)0); // minor version of numpy format
	if (ver == 1)
		Add(header, (uint16_t)dict.size());
	else
		Add(header, (uint32_t)dict.size());
	header.insert(header.end(), dict.begin(), dict.end());
	return header;
}

LPCSTR NpyArray::SaveNPY(std::string filename, bool bAppend) const
{
	FILE* fp;
	shape_t _shape;
	const shape_t* pShape;
	if (bAppend && (fp=fopen(filename.c_str(), "r+b")) != NULL) {
		// file exists, append to it; read the header, modify the array size
		char _type;
		size_t _wordSize;
		bool _fortranOrder;
		LPCSTR ret = ParseHeaderNPY(fp, _shape, _wordSize, _type, _fortranOrder);
		if (ret != NULL)
			return ret;
		ASSERT(!_fortranOrder);

		if (wordSize != _wordSize)
			return "error: npy_save word size";
		if (shape.size() != _shape.size())
			return "error: npy_save attempting to append mis-dimensioned data";

		for (size_t i = 1; i < shape.size(); i++) {
			if (shape[i] != _shape[i])
				return "error: npy_save attempting to append misshaped data";
		}
		_shape[0] += shape[0];
		pShape = &_shape;
	} else {
		// create a new file
		fp = fopen(filename.c_str(), "wb");
		pShape = &shape;
	}
	if (!fp)
		return "error: unable to open file";

	const std::vector<char> header = CreateHeaderNPY(*pShape, std::abs(type), wordSize);

	fseek(fp, 0, SEEK_SET);
	fwrite(header.data(), sizeof(char), header.size(), fp);
	fseek(fp, 0, SEEK_END);
	fwrite(Data(), wordSize, numValues, fp);
	fclose(fp);
	return NULL;
}

LPCSTR NpyArray::SaveNPZ(std::string zipname, std::string varname, bool bAppend) const
{
	FILE* fp;
	uint16_t nrecs = 0;
	size_t globalHeaderOffset = 0;
	std::vector<char> globalHeader;
	if (bAppend && (fp=fopen(zipname.c_str(), "r+b")) != NULL) {
		// zip file exists, add a new NPY array to it;
		// first read the footer and parse the offset and size of the global header
		// then read and store the global header;
		// the new data will be written at the start of the global header,
		// then append the global header and footer below it
		size_t globalHeaderSize;
		LPCSTR ret = ParseFooterZIP(fp, nrecs, globalHeaderSize, globalHeaderOffset);
		if (ret != NULL)
			return ret;
		fseek(fp, (long)globalHeaderOffset, SEEK_SET);
		globalHeader.resize(globalHeaderSize);
		size_t res = fread(globalHeader.data(), sizeof(char), globalHeaderSize, fp);
		if (res != globalHeaderSize)
			return "error: header read error while adding to existing zip";
		fseek(fp, (long)globalHeaderOffset, SEEK_SET);
	} else {
		fp = fopen(zipname.c_str(), "wb");
	}
	if (!fp)
		return "error: unable to open file";

	const std::vector<char> npyHeader = CreateHeaderNPY(shape, std::abs(type), wordSize);
	const size_t nbytes = SizeBytes() + npyHeader.size();

	// get the CRC of the data to be added
	uint32_t crc = crc32(0L, (uint8_t*)npyHeader.data(), (uLong)npyHeader.size());
	crc = crc32(crc, Data(), (uLong)SizeBytes());

	// append NPY extension
	varname += ".npy";

	// build the local header
	std::vector<char> localHeader;
	Add(localHeader, "PK"); // first part of signature
	Add(localHeader, (uint16_t)0x0403); // second part of signature
	Add(localHeader, (uint16_t)20); // min version to extract
	Add(localHeader, (uint16_t)0); // general purpose bit flag
	Add(localHeader, (uint16_t)0); // compression method
	Add(localHeader, (uint16_t)0); // file last mod time
	Add(localHeader, (uint16_t)0); // file last mod date
	Add(localHeader, (uint32_t)crc); // CRC
	Add(localHeader, (uint32_t)nbytes); // compressed size
	Add(localHeader, (uint32_t)nbytes); // uncompressed size
	Add(localHeader, (uint16_t)varname.size()); // variable name length
	Add(localHeader, (uint16_t)0); // extra field length
	Add(localHeader, varname);

	// build global header
	Add(globalHeader, "PK"); // first part of signature
	Add(globalHeader, (uint16_t)0x0201); // second part of signature
	Add(globalHeader, (uint16_t)20); // version made by
	globalHeader.insert(globalHeader.end(), localHeader.begin()+4, localHeader.begin()+30);
	Add(globalHeader, (uint16_t)0); // file comment length
	Add(globalHeader, (uint16_t)0); // disk number where file starts
	Add(globalHeader, (uint16_t)0); // internal file attributes
	Add(globalHeader, (uint32_t)0); // external file attributes
	Add(globalHeader, (uint32_t)globalHeaderOffset); // relative offset of local file header, since it begins where the global header used to begin
	Add(globalHeader, varname);

	// build footer
	std::vector<char> footer;
	Add(footer, "PK"); // first part of signature
	Add(footer, (uint16_t)0x0605); // second part of signature
	Add(footer, (uint16_t)0); // number of this disk
	Add(footer, (uint16_t)0); // disk where footer starts
	Add(footer, (uint16_t)(nrecs+1)); // number of records on this disk
	Add(footer, (uint16_t)(nrecs+1)); // total number of records
	Add(footer, (uint32_t)globalHeader.size()); // number of bytes of global headers
	Add(footer, (uint32_t)(globalHeaderOffset + nbytes + localHeader.size())); // offset of start of global headers, since global header now starts after newly written array
	Add(footer, (uint16_t)0); // zip file comment length

	// write everything
	fwrite(localHeader.data(), sizeof(char), localHeader.size(), fp);
	fwrite(npyHeader.data(), sizeof(char), npyHeader.size(), fp);
	fwrite(Data(), wordSize, numValues, fp);
	fwrite(globalHeader.data(), sizeof(char), globalHeader.size(), fp);
	fwrite(footer.data(), sizeof(char), footer.size(), fp);
	fclose(fp);
	return NULL;
}
/*----------------------------------------------------------------*/



// tools
char NpyArray::GetTypeChar(const std::type_info& t)
{
	if (t == typeid(float)) return 'f';
	if (t == typeid(double)) return 'f';
	if (t == typeid(long double)) return 'f';

	if (t == typeid(int)) return 'i';
	if (t == typeid(char)) return 'i';
	if (t == typeid(short)) return 'i';
	if (t == typeid(long)) return 'i';
	if (t == typeid(long long)) return 'i';

	if (t == typeid(uint8_t)) return 'u';
	if (t == typeid(unsigned short)) return 'u';
	if (t == typeid(unsigned long)) return 'u';
	if (t == typeid(unsigned long long)) return 'u';
	if (t == typeid(unsigned int)) return 'u';

	if (t == typeid(bool)) return 'b';

	if (t == typeid(std::complex<float>)) return 'c';
	if (t == typeid(std::complex<double>)) return 'c';
	if (t == typeid(std::complex<long double>)) return 'c';

	return '?';
}

const std::type_info& NpyArray::GetTypeInfo(char t, size_t s)
{
	switch (t) {
	case 'f': switch (s) {
		case  4: return typeid(float);
		case  8: return typeid(double);
		case 16: return typeid(long double);
		} break;
	case 'i': switch (s) {
		case  1: return typeid(char);
		case  2: return typeid(short);
		case  4: return typeid(int);
		case  8: return typeid(long);
		case 16: return typeid(long long);
		} break;
	case 'u': switch (s) {
		case  1: return typeid(unsigned char);
		case  2: return typeid(unsigned short);
		case  4: return typeid(unsigned int);
		case  8: return typeid(unsigned long);
		case 16: return typeid(unsigned long long);
		} break;
	case 'c': switch (s) {
		case  8: return typeid(std::complex<float>);
		case 16: return typeid(std::complex<double>);
		case 32: return typeid(std::complex<long double>);
		} break;
	case 'b': return typeid(bool);
	}
	return typeid(void);
}
/*----------------------------------------------------------------*/
std::vector<std::string> NpyArray::StringVector() const
{
	std::vector<std::string> vec;
	if (!IsStringType())
		return vec;
	vec.reserve(numValues);
	if (type == 'S' || type == 'a') {
		// Zero-terminated bytes strings (but fixed width field)
		const char* p = Data<char>();
		const size_t stride = SizeValueBytes();
		for (size_t i = 0; i < numValues; ++i) {
			const char* val = p + i * stride;
			size_t len = 0;
			while (len < stride && val[len] != '\0') len++;
			vec.emplace_back(val, len);
		}
	}
	else if (type == 'U') {
		const uint32_t* p = Data<uint32_t>();
		const size_t lenChars = wordSize / 4; // wordSize is now bytes, convert back to chars
		// Each string is fixed width of 'lenChars' characters (uint32)
		// Stride is lenChars for uint32 pointer
		for (size_t i = 0; i < numValues; ++i) {
			std::string s;
			const uint32_t* val = p + i * lenChars;
			for (size_t c = 0; c < lenChars; ++c) {
				uint32_t cp = val[c];
				if (cp == 0) break; // Null termination assumption
				// Convert UTF-32 code point to UTF-8
				if (cp < 0x80) s += (char)cp;
				else if (cp < 0x800) {
					s += (char)(0xC0 | (cp >> 6));
					s += (char)(0x80 | (cp & 0x3F));
				}
				else if (cp < 0x10000) {
					s += (char)(0xE0 | (cp >> 12));
					s += (char)(0x80 | ((cp >> 6) & 0x3F));
					s += (char)(0x80 | (cp & 0x3F));
				}
				else {
					s += (char)(0xF0 | (cp >> 18));
					s += (char)(0x80 | ((cp >> 12) & 0x3F));
					s += (char)(0x80 | ((cp >> 6) & 0x3F));
					s += (char)(0x80 | (cp & 0x3F));
				}
			}
			vec.push_back(std::move(s));
		}
	}

	return vec;
}

std::string NpyArray::PrintInfo(const std::string& name) const
{
	std::ostringstream ss;
	if (!name.empty())
		ss << "Array: " << name << "\n";
	ss << "  Dimensions:";
	for (size_t s: Shape())
		ss << " " << s;
	ss << "\n";
	ss << "  Number of values: " << NumValue() << "\n";
	ss << "  Size in bytes: " << SizeBytes() << "\n";
	const std::type_info& vt = ValueType();
	if (typeid(int) == vt)
		ss << "  Value type: int\n";
	if (typeid(float) == vt)
		ss << "  Value type: float\n";
	ss << "  Values order: " << (ColMajor() ? "col-major\n" : "row-major\n");

	if (IsStringType()) {
		ss << "  Value type: string\n";
		ss << "  Values: ";
		const std::vector<std::string> vec = StringVector();
		for (size_t i = 0; i < vec.size(); ++i) {
			if (i > 0) ss << ", ";
			ss << "\"" << vec[i] << "\"";
		}
		ss << "\n";
	}
	return ss.str();
}