Generated by Cython 3.2.8
Yellow lines hint at Python interaction.
Click on a line that starts with a "+" to see the C code that Cython generated for it.
Raw output: _c_build_dictionary.c
+001: # cython: language_level=3, boundscheck=False, wraparound=False
__pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
002: """Cython-accelerated inner loops for dictionary generation."""
003:
004: from cpython.exc cimport PyErr_CheckSignals
005:
006:
+007: def score_substrings(
/* Python wrapper */ static PyObject *__pyx_pw_4tamp_19_c_build_dictionary_1score_substrings(PyObject *__pyx_self, #if CYTHON_METH_FASTCALL PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds #else PyObject *__pyx_args, PyObject *__pyx_kwds #endif ); /*proto*/ PyDoc_STRVAR(__pyx_doc_4tamp_19_c_build_dictionary_score_substrings, "Count all substring occurrences and return scores dict.\n\n Retained for unit testing (TestScoreSubstrings). The main pipeline\n uses ``score_and_multi_frag`` which adds bottom-up pruning and\n multi-fragment tracking.\n\n Combines counting and scoring into a single Cython pass.\n Returns dict mapping substring -> total bits saved across corpus.\n "); static PyMethodDef __pyx_mdef_4tamp_19_c_build_dictionary_1score_substrings = {"score_substrings", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4tamp_19_c_build_dictionary_1score_substrings, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4tamp_19_c_build_dictionary_score_substrings}; static PyObject *__pyx_pw_4tamp_19_c_build_dictionary_1score_substrings(PyObject *__pyx_self, #if CYTHON_METH_FASTCALL PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds #else PyObject *__pyx_args, PyObject *__pyx_kwds #endif ) { PyObject *__pyx_v_corpus = 0; int __pyx_v_min_length; int __pyx_v_max_length; int __pyx_v_window_size; int __pyx_v_window_bits; int __pyx_v_literal_bits; PyObject *__pyx_v_huffman_bits = 0; #if !CYTHON_METH_FASTCALL CYTHON_UNUSED Py_ssize_t __pyx_nargs; #endif CYTHON_UNUSED PyObject *const *__pyx_kwvalues; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("score_substrings (wrapper)", 0); #if !CYTHON_METH_FASTCALL #if CYTHON_ASSUME_SAFE_SIZE __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); #else __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; #endif #endif __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); { PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_corpus,&__pyx_mstate_global->__pyx_n_u_min_length,&__pyx_mstate_global->__pyx_n_u_max_length,&__pyx_mstate_global->__pyx_n_u_window_size,&__pyx_mstate_global->__pyx_n_u_window_bits,&__pyx_mstate_global->__pyx_n_u_literal_bits,&__pyx_mstate_global->__pyx_n_u_huffman_bits,0}; PyObject* values[7] = {0,0,0,0,0,0,0}; const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 7, __pyx_L3_error) if (__pyx_kwds_len > 0) { switch (__pyx_nargs) { case 7: values[6] = __Pyx_ArgRef_FASTCALL(__pyx_args, 6); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[6])) __PYX_ERR(0, 7, __pyx_L3_error) CYTHON_FALLTHROUGH; case 6: values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 7, __pyx_L3_error) CYTHON_FALLTHROUGH; case 5: values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 7, __pyx_L3_error) CYTHON_FALLTHROUGH; case 4: values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 7, __pyx_L3_error) CYTHON_FALLTHROUGH; case 3: values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 7, __pyx_L3_error) CYTHON_FALLTHROUGH; case 2: values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 7, __pyx_L3_error) CYTHON_FALLTHROUGH; case 1: values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 7, __pyx_L3_error) CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } const Py_ssize_t kwd_pos_args = __pyx_nargs; if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "score_substrings", 0) < (0)) __PYX_ERR(0, 7, __pyx_L3_error) for (Py_ssize_t i = __pyx_nargs; i < 7; i++) { if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("score_substrings", 1, 7, 7, i); __PYX_ERR(0, 7, __pyx_L3_error) } } } else if (unlikely(__pyx_nargs != 7)) { goto __pyx_L5_argtuple_error; } else { values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 7, __pyx_L3_error) values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 7, __pyx_L3_error) values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 7, __pyx_L3_error) values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 7, __pyx_L3_error) values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 7, __pyx_L3_error) values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 7, __pyx_L3_error) values[6] = __Pyx_ArgRef_FASTCALL(__pyx_args, 6); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[6])) __PYX_ERR(0, 7, __pyx_L3_error) } __pyx_v_corpus = ((PyObject*)values[0]); __pyx_v_min_length = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_min_length == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 9, __pyx_L3_error) __pyx_v_max_length = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_max_length == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 10, __pyx_L3_error) __pyx_v_window_size = __Pyx_PyLong_As_int(values[3]); if (unlikely((__pyx_v_window_size == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 11, __pyx_L3_error) __pyx_v_window_bits = __Pyx_PyLong_As_int(values[4]); if (unlikely((__pyx_v_window_bits == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 12, __pyx_L3_error) __pyx_v_literal_bits = __Pyx_PyLong_As_int(values[5]); if (unlikely((__pyx_v_literal_bits == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 13, __pyx_L3_error) __pyx_v_huffman_bits = ((PyObject*)values[6]); } goto __pyx_L6_skip; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("score_substrings", 1, 7, 7, __pyx_nargs); __PYX_ERR(0, 7, __pyx_L3_error) __pyx_L6_skip:; goto __pyx_L4_argument_unpacking_done; __pyx_L3_error:; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } __Pyx_AddTraceback("tamp._c_build_dictionary.score_substrings", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_corpus), (&PyList_Type), 1, "corpus", 1))) __PYX_ERR(0, 8, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_huffman_bits), (&PyBytes_Type), 1, "huffman_bits", 1))) __PYX_ERR(0, 14, __pyx_L1_error) __pyx_r = __pyx_pf_4tamp_19_c_build_dictionary_score_substrings(__pyx_self, __pyx_v_corpus, __pyx_v_min_length, __pyx_v_max_length, __pyx_v_window_size, __pyx_v_window_bits, __pyx_v_literal_bits, __pyx_v_huffman_bits); int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; /* function exit code */ goto __pyx_L0; __pyx_L1_error:; __pyx_r = NULL; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } goto __pyx_L7_cleaned_up; __pyx_L0:; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } __pyx_L7_cleaned_up:; __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_pf_4tamp_19_c_build_dictionary_score_substrings(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_corpus, int __pyx_v_min_length, int __pyx_v_max_length, int __pyx_v_window_size, int __pyx_v_window_bits, int __pyx_v_literal_bits, PyObject *__pyx_v_huffman_bits) { PyObject *__pyx_v_counts = 0; PyObject *__pyx_v_sample = 0; PyObject *__pyx_v_sub = 0; int __pyx_v_sample_len; int __pyx_v_length; int __pyx_v_start; int __pyx_v_count; int __pyx_v_capped_max; PyObject *__pyx_v_scores = 0; int __pyx_v_match_len; int __pyx_v_i; double __pyx_v_literal_cost; double __pyx_v_match_cost; double __pyx_v_bits_saved; int __pyx_v_huff_len; unsigned char const *__pyx_v_huff; PyObject *__pyx_r = NULL; /* … */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_3); __Pyx_XDECREF(__pyx_t_13); __Pyx_XDECREF(__pyx_t_14); __Pyx_AddTraceback("tamp._c_build_dictionary.score_substrings", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XDECREF(__pyx_v_counts); __Pyx_XDECREF(__pyx_v_sample); __Pyx_XDECREF(__pyx_v_sub); __Pyx_XDECREF(__pyx_v_scores); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* … */ __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4tamp_19_c_build_dictionary_1score_substrings, 0, __pyx_mstate_global->__pyx_n_u_score_substrings, NULL, __pyx_mstate_global->__pyx_n_u_tamp__c_build_dictionary, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 7, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); #endif if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_score_substrings, __pyx_t_2) < (0)) __PYX_ERR(0, 7, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
008: list corpus,
009: int min_length,
010: int max_length,
011: int window_size,
012: int window_bits,
013: int literal_bits,
014: bytes huffman_bits,
015: ):
016: """Count all substring occurrences and return scores dict.
017:
018: Retained for unit testing (TestScoreSubstrings). The main pipeline
019: uses ``score_and_multi_frag`` which adds bottom-up pruning and
020: multi-fragment tracking.
021:
022: Combines counting and scoring into a single Cython pass.
023: Returns dict mapping substring -> total bits saved across corpus.
024: """
+025: cdef dict counts = {}
__pyx_t_1 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_counts = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
026: cdef bytes sample, sub
027: cdef int sample_len, length, start, count
028: cdef int capped_max
029:
+030: for sample in corpus:
if (unlikely(__pyx_v_corpus == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 30, __pyx_L1_error)
}
__pyx_t_1 = __pyx_v_corpus; __Pyx_INCREF(__pyx_t_1);
__pyx_t_2 = 0;
for (;;) {
{
Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_1);
#if !CYTHON_ASSUME_SAFE_SIZE
if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 30, __pyx_L1_error)
#endif
if (__pyx_t_2 >= __pyx_temp) break;
}
__pyx_t_3 = __Pyx_PyList_GetItemRefFast(__pyx_t_1, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference);
++__pyx_t_2;
if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 30, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_3))) __PYX_ERR(0, 30, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_sample, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
/* … */
__pyx_L3_continue:;
}
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+031: PyErr_CheckSignals()
__pyx_t_4 = PyErr_CheckSignals(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(0, 31, __pyx_L1_error)
+032: if len(sample) > window_size:
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 32, __pyx_L1_error)
}
__pyx_t_5 = __Pyx_PyBytes_GET_SIZE(__pyx_v_sample); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 32, __pyx_L1_error)
__pyx_t_6 = (__pyx_t_5 > __pyx_v_window_size);
if (__pyx_t_6) {
/* … */
}
+033: sample = sample[:window_size]
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 33, __pyx_L1_error)
}
__pyx_t_3 = PySequence_GetSlice(__pyx_v_sample, 0, __pyx_v_window_size); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 33, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF_SET(__pyx_v_sample, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
+034: sample_len = len(sample)
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 34, __pyx_L1_error)
}
__pyx_t_5 = __Pyx_PyBytes_GET_SIZE(__pyx_v_sample); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 34, __pyx_L1_error)
__pyx_v_sample_len = __pyx_t_5;
+035: if sample_len == 0:
__pyx_t_6 = (__pyx_v_sample_len == 0);
if (__pyx_t_6) {
/* … */
}
+036: continue
goto __pyx_L3_continue;
+037: capped_max = min(max_length + 1, sample_len + 1)
__pyx_t_7 = (__pyx_v_sample_len + 1);
__pyx_t_8 = (__pyx_v_max_length + 1);
__pyx_t_6 = (__pyx_t_7 < __pyx_t_8);
if (__pyx_t_6) {
__pyx_t_9 = __pyx_t_7;
} else {
__pyx_t_9 = __pyx_t_8;
}
__pyx_v_capped_max = __pyx_t_9;
+038: for length in range(min_length, capped_max):
__pyx_t_4 = __pyx_v_capped_max;
__pyx_t_10 = __pyx_t_4;
for (__pyx_t_11 = __pyx_v_min_length; __pyx_t_11 < __pyx_t_10; __pyx_t_11+=1) {
__pyx_v_length = __pyx_t_11;
+039: for start in range(sample_len - length + 1):
__pyx_t_9 = ((__pyx_v_sample_len - __pyx_v_length) + 1);
__pyx_t_7 = __pyx_t_9;
for (__pyx_t_12 = 0; __pyx_t_12 < __pyx_t_7; __pyx_t_12+=1) {
__pyx_v_start = __pyx_t_12;
+040: sub = sample[start : start + length]
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 40, __pyx_L1_error)
}
__pyx_t_3 = PySequence_GetSlice(__pyx_v_sample, __pyx_v_start, (__pyx_v_start + __pyx_v_length)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 40, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_XDECREF_SET(__pyx_v_sub, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
+041: if sub in counts:
__pyx_t_6 = (__Pyx_PyDict_ContainsTF(__pyx_v_sub, __pyx_v_counts, Py_EQ)); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 41, __pyx_L1_error) if (__pyx_t_6) { /* … */ goto __pyx_L11; }
+042: counts[sub] += 1
__Pyx_INCREF(__pyx_v_sub); __pyx_t_13 = __pyx_v_sub; __pyx_t_3 = __Pyx_PyDict_GetItem(__pyx_v_counts, __pyx_t_13); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 42, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_14 = __Pyx_PyLong_AddObjC(__pyx_t_3, __pyx_mstate_global->__pyx_int_1, 1, 1, 0); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 42, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_14); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (unlikely((PyDict_SetItem(__pyx_v_counts, __pyx_t_13, __pyx_t_14) < 0))) __PYX_ERR(0, 42, __pyx_L1_error) __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
043: else:
+044: counts[sub] = 1
/*else*/ {
if (unlikely((PyDict_SetItem(__pyx_v_counts, __pyx_v_sub, __pyx_mstate_global->__pyx_int_1) < 0))) __PYX_ERR(0, 44, __pyx_L1_error)
}
__pyx_L11:;
}
}
045:
+046: if not counts:
__pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_v_counts); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 46, __pyx_L1_error) __pyx_t_15 = (!__pyx_t_6); if (__pyx_t_15) { /* … */ }
+047: return {}
__Pyx_XDECREF(__pyx_r); __pyx_t_1 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 47, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0;
048:
+049: cdef dict scores = {}
__pyx_t_1 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_scores = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
050: cdef int match_len, i
051: cdef double literal_cost, match_cost, bits_saved
+052: cdef int huff_len = len(huffman_bits)
if (unlikely(__pyx_v_huffman_bits == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 52, __pyx_L1_error)
}
__pyx_t_2 = __Pyx_PyBytes_GET_SIZE(__pyx_v_huffman_bits); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 52, __pyx_L1_error)
__pyx_v_huff_len = __pyx_t_2;
+053: cdef const unsigned char *huff = huffman_bits
if (unlikely(__pyx_v_huffman_bits == Py_None)) {
PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found");
__PYX_ERR(0, 53, __pyx_L1_error)
}
__pyx_t_16 = __Pyx_PyBytes_AsUString(__pyx_v_huffman_bits); if (unlikely((!__pyx_t_16) && PyErr_Occurred())) __PYX_ERR(0, 53, __pyx_L1_error)
__pyx_v_huff = __pyx_t_16;
054:
+055: for sub, count in counts.items():
__pyx_t_2 = 0; __pyx_t_14 = __Pyx_dict_iterator(__pyx_v_counts, 1, __pyx_mstate_global->__pyx_n_u_items, (&__pyx_t_5), (&__pyx_t_4)); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 55, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_14); __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = __pyx_t_14; __pyx_t_14 = 0; while (1) { __pyx_t_10 = __Pyx_dict_iter_next(__pyx_t_1, __pyx_t_5, &__pyx_t_2, &__pyx_t_14, &__pyx_t_3, NULL, __pyx_t_4); if (unlikely(__pyx_t_10 == 0)) break; if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 55, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_14); __Pyx_GOTREF(__pyx_t_3); if (!(likely(PyBytes_CheckExact(__pyx_t_14))||((__pyx_t_14) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_14))) __PYX_ERR(0, 55, __pyx_L1_error) __pyx_t_10 = __Pyx_PyLong_As_int(__pyx_t_3); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 55, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_XDECREF_SET(__pyx_v_sub, ((PyObject*)__pyx_t_14)); __pyx_t_14 = 0; __pyx_v_count = __pyx_t_10;
+056: match_len = len(sub)
if (unlikely(__pyx_v_sub == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 56, __pyx_L1_error)
}
__pyx_t_17 = __Pyx_PyBytes_GET_SIZE(__pyx_v_sub); if (unlikely(__pyx_t_17 == ((Py_ssize_t)-1))) __PYX_ERR(0, 56, __pyx_L1_error)
__pyx_v_match_len = __pyx_t_17;
+057: literal_cost = match_len * (1 + literal_bits)
__pyx_v_literal_cost = (__pyx_v_match_len * (1 + __pyx_v_literal_bits));
+058: i = match_len - min_length
__pyx_v_i = (__pyx_v_match_len - __pyx_v_min_length);
+059: if i < 0 or i >= huff_len:
__pyx_t_6 = (__pyx_v_i < 0);
if (!__pyx_t_6) {
} else {
__pyx_t_15 = __pyx_t_6;
goto __pyx_L17_bool_binop_done;
}
__pyx_t_6 = (__pyx_v_i >= __pyx_v_huff_len);
__pyx_t_15 = __pyx_t_6;
__pyx_L17_bool_binop_done:;
if (__pyx_t_15) {
/* … */
}
+060: continue
goto __pyx_L14_continue;
+061: match_cost = huff[i] + window_bits
__pyx_v_match_cost = ((__pyx_v_huff[__pyx_v_i]) + __pyx_v_window_bits);
+062: bits_saved = literal_cost - match_cost
__pyx_v_bits_saved = (__pyx_v_literal_cost - __pyx_v_match_cost);
+063: if bits_saved > 0:
__pyx_t_15 = (__pyx_v_bits_saved > 0.0);
if (__pyx_t_15) {
/* … */
}
__pyx_L14_continue:;
}
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+064: scores[sub] = count * bits_saved
__pyx_t_3 = PyFloat_FromDouble((__pyx_v_count * __pyx_v_bits_saved)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 64, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (unlikely((PyDict_SetItem(__pyx_v_scores, __pyx_v_sub, __pyx_t_3) < 0))) __PYX_ERR(0, 64, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
065:
+066: return scores
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_scores); __pyx_r = __pyx_v_scores; goto __pyx_L0;
067:
068:
+069: def score_and_multi_frag(
/* Python wrapper */ static PyObject *__pyx_pw_4tamp_19_c_build_dictionary_3score_and_multi_frag(PyObject *__pyx_self, #if CYTHON_METH_FASTCALL PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds #else PyObject *__pyx_args, PyObject *__pyx_kwds #endif ); /*proto*/ PyDoc_STRVAR(__pyx_doc_4tamp_19_c_build_dictionary_2score_and_multi_frag, "Score substrings and identify multi-fragment substrings.\n\n Uses bottom-up pruning: a substring of length L can only appear in\n 2+ samples if its length-(L-1) prefix does too. Starts at min_length,\n keeps only frequent prefixes, and extends incrementally. This avoids\n enumerating the vast majority of long unique substrings.\n\n Parameters\n ----------\n bits_saved_table\n Precomputed list of bits saved for each match length.\n Indexed by (length - min_length). Covers both basic and\n extended match encodings.\n\n Returns (scores_dict, multi_frag_set).\n "); static PyMethodDef __pyx_mdef_4tamp_19_c_build_dictionary_3score_and_multi_frag = {"score_and_multi_frag", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4tamp_19_c_build_dictionary_3score_and_multi_frag, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4tamp_19_c_build_dictionary_2score_and_multi_frag}; static PyObject *__pyx_pw_4tamp_19_c_build_dictionary_3score_and_multi_frag(PyObject *__pyx_self, #if CYTHON_METH_FASTCALL PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds #else PyObject *__pyx_args, PyObject *__pyx_kwds #endif ) { PyObject *__pyx_v_corpus = 0; int __pyx_v_min_length; int __pyx_v_max_length; int __pyx_v_window_size; PyObject *__pyx_v_bits_saved_table = 0; int __pyx_v_multi_frag_min_length; #if !CYTHON_METH_FASTCALL CYTHON_UNUSED Py_ssize_t __pyx_nargs; #endif CYTHON_UNUSED PyObject *const *__pyx_kwvalues; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("score_and_multi_frag (wrapper)", 0); #if !CYTHON_METH_FASTCALL #if CYTHON_ASSUME_SAFE_SIZE __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); #else __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; #endif #endif __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); { PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_corpus,&__pyx_mstate_global->__pyx_n_u_min_length,&__pyx_mstate_global->__pyx_n_u_max_length,&__pyx_mstate_global->__pyx_n_u_window_size,&__pyx_mstate_global->__pyx_n_u_bits_saved_table,&__pyx_mstate_global->__pyx_n_u_multi_frag_min_length,0}; PyObject* values[6] = {0,0,0,0,0,0}; const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 69, __pyx_L3_error) if (__pyx_kwds_len > 0) { switch (__pyx_nargs) { case 6: values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 69, __pyx_L3_error) CYTHON_FALLTHROUGH; case 5: values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 69, __pyx_L3_error) CYTHON_FALLTHROUGH; case 4: values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 69, __pyx_L3_error) CYTHON_FALLTHROUGH; case 3: values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 69, __pyx_L3_error) CYTHON_FALLTHROUGH; case 2: values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 69, __pyx_L3_error) CYTHON_FALLTHROUGH; case 1: values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 69, __pyx_L3_error) CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } const Py_ssize_t kwd_pos_args = __pyx_nargs; if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "score_and_multi_frag", 0) < (0)) __PYX_ERR(0, 69, __pyx_L3_error) for (Py_ssize_t i = __pyx_nargs; i < 6; i++) { if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("score_and_multi_frag", 1, 6, 6, i); __PYX_ERR(0, 69, __pyx_L3_error) } } } else if (unlikely(__pyx_nargs != 6)) { goto __pyx_L5_argtuple_error; } else { values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 69, __pyx_L3_error) values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 69, __pyx_L3_error) values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 69, __pyx_L3_error) values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 69, __pyx_L3_error) values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 69, __pyx_L3_error) values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 69, __pyx_L3_error) } __pyx_v_corpus = ((PyObject*)values[0]); __pyx_v_min_length = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_min_length == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L3_error) __pyx_v_max_length = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_max_length == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 72, __pyx_L3_error) __pyx_v_window_size = __Pyx_PyLong_As_int(values[3]); if (unlikely((__pyx_v_window_size == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 73, __pyx_L3_error) __pyx_v_bits_saved_table = ((PyObject*)values[4]); __pyx_v_multi_frag_min_length = __Pyx_PyLong_As_int(values[5]); if (unlikely((__pyx_v_multi_frag_min_length == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 75, __pyx_L3_error) } goto __pyx_L6_skip; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("score_and_multi_frag", 1, 6, 6, __pyx_nargs); __PYX_ERR(0, 69, __pyx_L3_error) __pyx_L6_skip:; goto __pyx_L4_argument_unpacking_done; __pyx_L3_error:; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } __Pyx_AddTraceback("tamp._c_build_dictionary.score_and_multi_frag", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_corpus), (&PyList_Type), 1, "corpus", 1))) __PYX_ERR(0, 70, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bits_saved_table), (&PyList_Type), 1, "bits_saved_table", 1))) __PYX_ERR(0, 74, __pyx_L1_error) __pyx_r = __pyx_pf_4tamp_19_c_build_dictionary_2score_and_multi_frag(__pyx_self, __pyx_v_corpus, __pyx_v_min_length, __pyx_v_max_length, __pyx_v_window_size, __pyx_v_bits_saved_table, __pyx_v_multi_frag_min_length); int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; /* function exit code */ goto __pyx_L0; __pyx_L1_error:; __pyx_r = NULL; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } goto __pyx_L7_cleaned_up; __pyx_L0:; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } __pyx_L7_cleaned_up:; __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_pf_4tamp_19_c_build_dictionary_2score_and_multi_frag(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_corpus, int __pyx_v_min_length, int __pyx_v_max_length, int __pyx_v_window_size, PyObject *__pyx_v_bits_saved_table, int __pyx_v_multi_frag_min_length) { PyObject *__pyx_v_samples = 0; PyObject *__pyx_v_s = 0; PyObject *__pyx_v_scores = 0; PyObject *__pyx_v_multi_frag = 0; int __pyx_v_table_len; PyObject *__pyx_v_sample_counts = 0; PyObject *__pyx_v_sample_subs = 0; PyObject *__pyx_v_freq = 0; PyObject *__pyx_v_sample = 0; PyObject *__pyx_v_sub = 0; PyObject *__pyx_v_prefix = 0; int __pyx_v_sample_len; int __pyx_v_start; int __pyx_v_length; int __pyx_v_sc; int __pyx_v_i; double __pyx_v_bits_saved; PyObject *__pyx_r = NULL; /* … */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_3); __Pyx_XDECREF(__pyx_t_8); __Pyx_XDECREF(__pyx_t_14); __Pyx_XDECREF(__pyx_t_15); __Pyx_AddTraceback("tamp._c_build_dictionary.score_and_multi_frag", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XDECREF(__pyx_v_samples); __Pyx_XDECREF(__pyx_v_s); __Pyx_XDECREF(__pyx_v_scores); __Pyx_XDECREF(__pyx_v_multi_frag); __Pyx_XDECREF(__pyx_v_sample_counts); __Pyx_XDECREF(__pyx_v_sample_subs); __Pyx_XDECREF(__pyx_v_freq); __Pyx_XDECREF(__pyx_v_sample); __Pyx_XDECREF(__pyx_v_sub); __Pyx_XDECREF(__pyx_v_prefix); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* … */ __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4tamp_19_c_build_dictionary_3score_and_multi_frag, 0, __pyx_mstate_global->__pyx_n_u_score_and_multi_frag, NULL, __pyx_mstate_global->__pyx_n_u_tamp__c_build_dictionary, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 69, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); #endif if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_score_and_multi_frag, __pyx_t_2) < (0)) __PYX_ERR(0, 69, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
070: list corpus,
071: int min_length,
072: int max_length,
073: int window_size,
074: list bits_saved_table,
075: int multi_frag_min_length,
076: ):
077: """Score substrings and identify multi-fragment substrings.
078:
079: Uses bottom-up pruning: a substring of length L can only appear in
080: 2+ samples if its length-(L-1) prefix does too. Starts at min_length,
081: keeps only frequent prefixes, and extends incrementally. This avoids
082: enumerating the vast majority of long unique substrings.
083:
084: Parameters
085: ----------
086: bits_saved_table
087: Precomputed list of bits saved for each match length.
088: Indexed by (length - min_length). Covers both basic and
089: extended match encodings.
090:
091: Returns (scores_dict, multi_frag_set).
092: """
+093: cdef list samples = []
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 93, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_samples = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
094: cdef bytes s
+095: for s in corpus:
if (unlikely(__pyx_v_corpus == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 95, __pyx_L1_error)
}
__pyx_t_1 = __pyx_v_corpus; __Pyx_INCREF(__pyx_t_1);
__pyx_t_2 = 0;
for (;;) {
{
Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_1);
#if !CYTHON_ASSUME_SAFE_SIZE
if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 95, __pyx_L1_error)
#endif
if (__pyx_t_2 >= __pyx_temp) break;
}
__pyx_t_3 = __Pyx_PyList_GetItemRefFast(__pyx_t_1, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference);
++__pyx_t_2;
if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 95, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_3))) __PYX_ERR(0, 95, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_s, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+096: if len(s) > window_size:
if (unlikely(__pyx_v_s == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 96, __pyx_L1_error)
}
__pyx_t_4 = __Pyx_PyBytes_GET_SIZE(__pyx_v_s); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(0, 96, __pyx_L1_error)
__pyx_t_5 = (__pyx_t_4 > __pyx_v_window_size);
if (__pyx_t_5) {
/* … */
}
+097: s = s[:window_size]
if (unlikely(__pyx_v_s == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 97, __pyx_L1_error)
}
__pyx_t_3 = PySequence_GetSlice(__pyx_v_s, 0, __pyx_v_window_size); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 97, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF_SET(__pyx_v_s, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
+098: if len(s) > 0:
if (unlikely(__pyx_v_s == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 98, __pyx_L1_error)
}
__pyx_t_4 = __Pyx_PyBytes_GET_SIZE(__pyx_v_s); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(0, 98, __pyx_L1_error)
__pyx_t_5 = (__pyx_t_4 > 0);
if (__pyx_t_5) {
/* … */
}
+099: samples.append(s)
__pyx_t_6 = __Pyx_PyList_Append(__pyx_v_samples, __pyx_v_s); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 99, __pyx_L1_error)
100:
+101: if not samples:
{
Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_v_samples);
if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 101, __pyx_L1_error)
__pyx_t_5 = (__pyx_temp != 0);
}
__pyx_t_7 = (!__pyx_t_5);
if (__pyx_t_7) {
/* … */
}
+102: return {}, set()
__Pyx_XDECREF(__pyx_r); __pyx_t_1 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 102, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = PySet_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 102, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 102, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_1); if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 102, __pyx_L1_error); __Pyx_GIVEREF(__pyx_t_3); if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 102, __pyx_L1_error); __pyx_t_1 = 0; __pyx_t_3 = 0; __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L0;
103:
+104: cdef dict scores = {}
__pyx_t_8 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 104, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_v_scores = ((PyObject*)__pyx_t_8); __pyx_t_8 = 0;
+105: cdef set multi_frag = set()
__pyx_t_8 = PySet_New(0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 105, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_v_multi_frag = ((PyObject*)__pyx_t_8); __pyx_t_8 = 0;
+106: cdef int table_len = len(bits_saved_table)
if (unlikely(__pyx_v_bits_saved_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 106, __pyx_L1_error)
}
__pyx_t_2 = __Pyx_PyList_GET_SIZE(__pyx_v_bits_saved_table); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 106, __pyx_L1_error)
__pyx_v_table_len = __pyx_t_2;
107:
108: cdef dict sample_counts
109: cdef set sample_subs
110: cdef set freq
111: cdef bytes sample, sub, prefix
112: cdef int sample_len, start, length, sc, i
113: cdef double bits_saved
114:
115: # Bootstrap: enumerate all substrings at min_length.
116: # Track per-sample counts (not total occurrences) for scoring.
+117: sample_counts = {}
__pyx_t_8 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 117, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_v_sample_counts = ((PyObject*)__pyx_t_8); __pyx_t_8 = 0;
+118: for sample in samples:
__pyx_t_8 = __pyx_v_samples; __Pyx_INCREF(__pyx_t_8); __pyx_t_2 = 0; for (;;) { { Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_8); #if !CYTHON_ASSUME_SAFE_SIZE if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 118, __pyx_L1_error) #endif if (__pyx_t_2 >= __pyx_temp) break; } __pyx_t_3 = __Pyx_PyList_GetItemRefFast(__pyx_t_8, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference); ++__pyx_t_2; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 118, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_3))) __PYX_ERR(0, 118, __pyx_L1_error) __Pyx_XDECREF_SET(__pyx_v_sample, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; /* … */ } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+119: PyErr_CheckSignals()
__pyx_t_9 = PyErr_CheckSignals(); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 119, __pyx_L1_error)
+120: sample_len = len(sample)
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 120, __pyx_L1_error)
}
__pyx_t_4 = __Pyx_PyBytes_GET_SIZE(__pyx_v_sample); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(0, 120, __pyx_L1_error)
__pyx_v_sample_len = __pyx_t_4;
+121: sample_subs = set()
__pyx_t_3 = PySet_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 121, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_XDECREF_SET(__pyx_v_sample_subs, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0;
+122: for start in range(sample_len - min_length + 1):
__pyx_t_10 = ((__pyx_v_sample_len - __pyx_v_min_length) + 1);
__pyx_t_11 = __pyx_t_10;
for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_11; __pyx_t_9+=1) {
__pyx_v_start = __pyx_t_9;
+123: sample_subs.add(sample[start : start + min_length])
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 123, __pyx_L1_error)
}
__pyx_t_3 = PySequence_GetSlice(__pyx_v_sample, __pyx_v_start, (__pyx_v_start + __pyx_v_min_length)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 123, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_6 = PySet_Add(__pyx_v_sample_subs, __pyx_t_3); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 123, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
}
+124: for sub in sample_subs:
__pyx_t_4 = 0;
__pyx_t_1 = __Pyx_set_iterator(__pyx_v_sample_subs, 1, (&__pyx_t_12), (&__pyx_t_9)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 124, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_3);
__pyx_t_3 = __pyx_t_1;
__pyx_t_1 = 0;
while (1) {
__pyx_t_13 = __Pyx_set_iter_next(__pyx_t_3, __pyx_t_12, &__pyx_t_4, &__pyx_t_1, __pyx_t_9);
if (unlikely(__pyx_t_13 == 0)) break;
if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 124, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_1))) __PYX_ERR(0, 124, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_sub, ((PyObject*)__pyx_t_1));
__pyx_t_1 = 0;
+125: if sub in sample_counts:
__pyx_t_7 = (__Pyx_PyDict_ContainsTF(__pyx_v_sub, __pyx_v_sample_counts, Py_EQ)); if (unlikely((__pyx_t_7 < 0))) __PYX_ERR(0, 125, __pyx_L1_error) if (__pyx_t_7) { /* … */ goto __pyx_L15; }
+126: sample_counts[sub] += 1
__Pyx_INCREF(__pyx_v_sub); __pyx_t_14 = __pyx_v_sub; __pyx_t_1 = __Pyx_PyDict_GetItem(__pyx_v_sample_counts, __pyx_t_14); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 126, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_15 = __Pyx_PyLong_AddObjC(__pyx_t_1, __pyx_mstate_global->__pyx_int_1, 1, 1, 0); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 126, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (unlikely((PyDict_SetItem(__pyx_v_sample_counts, __pyx_t_14, __pyx_t_15) < 0))) __PYX_ERR(0, 126, __pyx_L1_error) __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
127: else:
+128: sample_counts[sub] = 1
/*else*/ {
if (unlikely((PyDict_SetItem(__pyx_v_sample_counts, __pyx_v_sub, __pyx_mstate_global->__pyx_int_1) < 0))) __PYX_ERR(0, 128, __pyx_L1_error)
}
__pyx_L15:;
}
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
129:
130: # Score and collect frequent prefixes.
+131: freq = set()
__pyx_t_8 = PySet_New(0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 131, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_v_freq = ((PyObject*)__pyx_t_8); __pyx_t_8 = 0;
+132: bits_saved = bits_saved_table[0] if table_len > 0 else 0
__pyx_t_7 = (__pyx_v_table_len > 0);
if (__pyx_t_7) {
if (unlikely(__pyx_v_bits_saved_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 132, __pyx_L1_error)
}
__pyx_t_17 = __Pyx_PyFloat_AsDouble(__Pyx_PyList_GET_ITEM(__pyx_v_bits_saved_table, 0)); if (unlikely((__pyx_t_17 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 132, __pyx_L1_error)
__pyx_t_16 = __pyx_t_17;
} else {
__pyx_t_16 = 0.0;
}
__pyx_v_bits_saved = __pyx_t_16;
+133: if bits_saved > 0:
__pyx_t_7 = (__pyx_v_bits_saved > 0.0);
if (__pyx_t_7) {
/* … */
goto __pyx_L17;
}
+134: for sub, sc in sample_counts.items():
__pyx_t_2 = 0;
__pyx_t_3 = __Pyx_dict_iterator(__pyx_v_sample_counts, 1, __pyx_mstate_global->__pyx_n_u_items, (&__pyx_t_12), (&__pyx_t_9)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 134, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_8);
__pyx_t_8 = __pyx_t_3;
__pyx_t_3 = 0;
while (1) {
__pyx_t_13 = __Pyx_dict_iter_next(__pyx_t_8, __pyx_t_12, &__pyx_t_2, &__pyx_t_3, &__pyx_t_15, NULL, __pyx_t_9);
if (unlikely(__pyx_t_13 == 0)) break;
if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 134, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_GOTREF(__pyx_t_15);
if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_3))) __PYX_ERR(0, 134, __pyx_L1_error)
__pyx_t_13 = __Pyx_PyLong_As_int(__pyx_t_15); if (unlikely((__pyx_t_13 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 134, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
__Pyx_XDECREF_SET(__pyx_v_sub, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
__pyx_v_sc = __pyx_t_13;
+135: if sc >= 2:
__pyx_t_7 = (__pyx_v_sc >= 2);
if (__pyx_t_7) {
/* … */
}
}
__Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+136: scores[sub] = sc * bits_saved
__pyx_t_15 = PyFloat_FromDouble((__pyx_v_sc * __pyx_v_bits_saved)); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); if (unlikely((PyDict_SetItem(__pyx_v_scores, __pyx_v_sub, __pyx_t_15) < 0))) __PYX_ERR(0, 136, __pyx_L1_error) __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
+137: freq.add(sub)
__pyx_t_6 = PySet_Add(__pyx_v_freq, __pyx_v_sub); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 137, __pyx_L1_error)
+138: if min_length >= multi_frag_min_length:
__pyx_t_7 = (__pyx_v_min_length >= __pyx_v_multi_frag_min_length);
if (__pyx_t_7) {
/* … */
}
+139: multi_frag.add(sub)
__pyx_t_6 = PySet_Add(__pyx_v_multi_frag, __pyx_v_sub); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 139, __pyx_L1_error)
140: else:
+141: for sub, sc in sample_counts.items():
/*else*/ {
__pyx_t_12 = 0;
__pyx_t_15 = __Pyx_dict_iterator(__pyx_v_sample_counts, 1, __pyx_mstate_global->__pyx_n_u_items, (&__pyx_t_2), (&__pyx_t_9)); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 141, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
__Pyx_XDECREF(__pyx_t_8);
__pyx_t_8 = __pyx_t_15;
__pyx_t_15 = 0;
while (1) {
__pyx_t_13 = __Pyx_dict_iter_next(__pyx_t_8, __pyx_t_2, &__pyx_t_12, &__pyx_t_15, &__pyx_t_3, NULL, __pyx_t_9);
if (unlikely(__pyx_t_13 == 0)) break;
if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 141, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
__Pyx_GOTREF(__pyx_t_3);
if (!(likely(PyBytes_CheckExact(__pyx_t_15))||((__pyx_t_15) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_15))) __PYX_ERR(0, 141, __pyx_L1_error)
__pyx_t_13 = __Pyx_PyLong_As_int(__pyx_t_3); if (unlikely((__pyx_t_13 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 141, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__Pyx_XDECREF_SET(__pyx_v_sub, ((PyObject*)__pyx_t_15));
__pyx_t_15 = 0;
__pyx_v_sc = __pyx_t_13;
+142: if sc >= 2:
__pyx_t_7 = (__pyx_v_sc >= 2);
if (__pyx_t_7) {
/* … */
}
}
__Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
}
__pyx_L17:;
+143: freq.add(sub)
__pyx_t_6 = PySet_Add(__pyx_v_freq, __pyx_v_sub); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 143, __pyx_L1_error)
144:
145: # Extend length by length, pruning by frequent prefixes.
+146: for length in range(min_length + 1, max_length + 1):
__pyx_t_10 = (__pyx_v_max_length + 1);
__pyx_t_11 = __pyx_t_10;
for (__pyx_t_9 = (__pyx_v_min_length + 1); __pyx_t_9 < __pyx_t_11; __pyx_t_9+=1) {
__pyx_v_length = __pyx_t_9;
+147: if not freq:
{
Py_ssize_t __pyx_temp = __Pyx_PySet_GET_SIZE(__pyx_v_freq);
if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 147, __pyx_L1_error)
__pyx_t_7 = (__pyx_temp != 0);
}
__pyx_t_5 = (!__pyx_t_7);
if (__pyx_t_5) {
/* … */
}
+148: break
goto __pyx_L26_break;
149:
+150: sample_counts = {}
__pyx_t_8 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF_SET(__pyx_v_sample_counts, ((PyObject*)__pyx_t_8)); __pyx_t_8 = 0;
+151: for sample in samples:
__pyx_t_8 = __pyx_v_samples; __Pyx_INCREF(__pyx_t_8); __pyx_t_2 = 0; for (;;) { { Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_8); #if !CYTHON_ASSUME_SAFE_SIZE if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 151, __pyx_L1_error) #endif if (__pyx_t_2 >= __pyx_temp) break; } __pyx_t_3 = __Pyx_PyList_GetItemRefFast(__pyx_t_8, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference); ++__pyx_t_2; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_3))) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_XDECREF_SET(__pyx_v_sample, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; /* … */ __pyx_L28_continue:; } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+152: PyErr_CheckSignals()
__pyx_t_13 = PyErr_CheckSignals(); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 152, __pyx_L1_error)
+153: sample_len = len(sample)
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 153, __pyx_L1_error)
}
__pyx_t_12 = __Pyx_PyBytes_GET_SIZE(__pyx_v_sample); if (unlikely(__pyx_t_12 == ((Py_ssize_t)-1))) __PYX_ERR(0, 153, __pyx_L1_error)
__pyx_v_sample_len = __pyx_t_12;
+154: if sample_len < length:
__pyx_t_5 = (__pyx_v_sample_len < __pyx_v_length);
if (__pyx_t_5) {
/* … */
}
+155: continue
goto __pyx_L28_continue;
+156: sample_subs = set()
__pyx_t_3 = PySet_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 156, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_XDECREF_SET(__pyx_v_sample_subs, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0;
+157: for start in range(sample_len - length + 1):
__pyx_t_18 = ((__pyx_v_sample_len - __pyx_v_length) + 1);
__pyx_t_19 = __pyx_t_18;
for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_19; __pyx_t_13+=1) {
__pyx_v_start = __pyx_t_13;
+158: prefix = sample[start : start + length - 1]
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 158, __pyx_L1_error)
}
__pyx_t_3 = PySequence_GetSlice(__pyx_v_sample, __pyx_v_start, ((__pyx_v_start + __pyx_v_length) - 1)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 158, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_XDECREF_SET(__pyx_v_prefix, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
+159: if prefix not in freq:
__pyx_t_5 = (__Pyx_PySet_ContainsTF(__pyx_v_prefix, __pyx_v_freq, Py_NE)); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 159, __pyx_L1_error) if (__pyx_t_5) { /* … */ }
+160: continue
goto __pyx_L31_continue;
+161: sample_subs.add(sample[start : start + length])
if (unlikely(__pyx_v_sample == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 161, __pyx_L1_error)
}
__pyx_t_3 = PySequence_GetSlice(__pyx_v_sample, __pyx_v_start, (__pyx_v_start + __pyx_v_length)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 161, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_6 = PySet_Add(__pyx_v_sample_subs, __pyx_t_3); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 161, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_L31_continue:;
}
+162: for sub in sample_subs:
__pyx_t_12 = 0;
__pyx_t_15 = __Pyx_set_iterator(__pyx_v_sample_subs, 1, (&__pyx_t_4), (&__pyx_t_13)); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 162, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
__Pyx_XDECREF(__pyx_t_3);
__pyx_t_3 = __pyx_t_15;
__pyx_t_15 = 0;
while (1) {
__pyx_t_20 = __Pyx_set_iter_next(__pyx_t_3, __pyx_t_4, &__pyx_t_12, &__pyx_t_15, __pyx_t_13);
if (unlikely(__pyx_t_20 == 0)) break;
if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 162, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
if (!(likely(PyBytes_CheckExact(__pyx_t_15))||((__pyx_t_15) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_15))) __PYX_ERR(0, 162, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_sub, ((PyObject*)__pyx_t_15));
__pyx_t_15 = 0;
+163: if sub in sample_counts:
__pyx_t_5 = (__Pyx_PyDict_ContainsTF(__pyx_v_sub, __pyx_v_sample_counts, Py_EQ)); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 163, __pyx_L1_error) if (__pyx_t_5) { /* … */ goto __pyx_L36; }
+164: sample_counts[sub] += 1
__Pyx_INCREF(__pyx_v_sub); __pyx_t_14 = __pyx_v_sub; __pyx_t_15 = __Pyx_PyDict_GetItem(__pyx_v_sample_counts, __pyx_t_14); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __pyx_t_1 = __Pyx_PyLong_AddObjC(__pyx_t_15, __pyx_mstate_global->__pyx_int_1, 1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; if (unlikely((PyDict_SetItem(__pyx_v_sample_counts, __pyx_t_14, __pyx_t_1) < 0))) __PYX_ERR(0, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
165: else:
+166: sample_counts[sub] = 1
/*else*/ {
if (unlikely((PyDict_SetItem(__pyx_v_sample_counts, __pyx_v_sub, __pyx_mstate_global->__pyx_int_1) < 0))) __PYX_ERR(0, 166, __pyx_L1_error)
}
__pyx_L36:;
}
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
167:
168: # Score this length and build new frequent set.
+169: freq = set()
__pyx_t_8 = PySet_New(0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 169, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF_SET(__pyx_v_freq, ((PyObject*)__pyx_t_8)); __pyx_t_8 = 0;
+170: i = length - min_length
__pyx_v_i = (__pyx_v_length - __pyx_v_min_length);
+171: bits_saved = bits_saved_table[i] if i < table_len else 0
__pyx_t_5 = (__pyx_v_i < __pyx_v_table_len);
if (__pyx_t_5) {
if (unlikely(__pyx_v_bits_saved_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 171, __pyx_L1_error)
}
__pyx_t_17 = __Pyx_PyFloat_AsDouble(__Pyx_PyList_GET_ITEM(__pyx_v_bits_saved_table, __pyx_v_i)); if (unlikely((__pyx_t_17 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 171, __pyx_L1_error)
__pyx_t_16 = __pyx_t_17;
} else {
__pyx_t_16 = 0.0;
}
__pyx_v_bits_saved = __pyx_t_16;
+172: for sub, sc in sample_counts.items():
__pyx_t_2 = 0;
__pyx_t_3 = __Pyx_dict_iterator(__pyx_v_sample_counts, 1, __pyx_mstate_global->__pyx_n_u_items, (&__pyx_t_4), (&__pyx_t_13)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 172, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_8);
__pyx_t_8 = __pyx_t_3;
__pyx_t_3 = 0;
while (1) {
__pyx_t_20 = __Pyx_dict_iter_next(__pyx_t_8, __pyx_t_4, &__pyx_t_2, &__pyx_t_3, &__pyx_t_1, NULL, __pyx_t_13);
if (unlikely(__pyx_t_20 == 0)) break;
if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 172, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_GOTREF(__pyx_t_1);
if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_3))) __PYX_ERR(0, 172, __pyx_L1_error)
__pyx_t_20 = __Pyx_PyLong_As_int(__pyx_t_1); if (unlikely((__pyx_t_20 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 172, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
__Pyx_XDECREF_SET(__pyx_v_sub, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
__pyx_v_sc = __pyx_t_20;
+173: if sc >= 2:
__pyx_t_5 = (__pyx_v_sc >= 2);
if (__pyx_t_5) {
/* … */
}
}
__Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
}
__pyx_L26_break:;
+174: if bits_saved > 0:
__pyx_t_5 = (__pyx_v_bits_saved > 0.0);
if (__pyx_t_5) {
/* … */
}
+175: scores[sub] = sc * bits_saved
__pyx_t_1 = PyFloat_FromDouble((__pyx_v_sc * __pyx_v_bits_saved)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 175, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (unlikely((PyDict_SetItem(__pyx_v_scores, __pyx_v_sub, __pyx_t_1) < 0))) __PYX_ERR(0, 175, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+176: if length >= multi_frag_min_length:
__pyx_t_5 = (__pyx_v_length >= __pyx_v_multi_frag_min_length);
if (__pyx_t_5) {
/* … */
}
+177: multi_frag.add(sub)
__pyx_t_6 = PySet_Add(__pyx_v_multi_frag, __pyx_v_sub); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 177, __pyx_L1_error)
+178: freq.add(sub)
__pyx_t_6 = PySet_Add(__pyx_v_freq, __pyx_v_sub); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 178, __pyx_L1_error)
179:
+180: return scores, multi_frag
__Pyx_XDECREF(__pyx_r); __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 180, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_v_scores); __Pyx_GIVEREF(__pyx_v_scores); if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_scores) != (0)) __PYX_ERR(0, 180, __pyx_L1_error); __Pyx_INCREF(__pyx_v_multi_frag); __Pyx_GIVEREF(__pyx_v_multi_frag); if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_v_multi_frag) != (0)) __PYX_ERR(0, 180, __pyx_L1_error); __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L0;
181:
182:
+183: def select_candidates(
/* Python wrapper */ static PyObject *__pyx_pw_4tamp_19_c_build_dictionary_5select_candidates(PyObject *__pyx_self, #if CYTHON_METH_FASTCALL PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds #else PyObject *__pyx_args, PyObject *__pyx_kwds #endif ); /*proto*/ PyDoc_STRVAR(__pyx_doc_4tamp_19_c_build_dictionary_4select_candidates, "Extract all valid entries from candidates until budget is exhausted.\n\n Iterates through candidates (sorted by score descending). For each\n valid candidate (exists in multi_frag_content, i.e. appears in 2+\n fragments), accepts it and removes all candidates that share\n content of >= overlap_threshold bytes. This prevents shifted\n duplicates like \"I DO NOT LIKE \" and \" DO NOT LIKE THE\" from\n both being selected.\n\n Returns a list of accepted entry bytes.\n "); static PyMethodDef __pyx_mdef_4tamp_19_c_build_dictionary_5select_candidates = {"select_candidates", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4tamp_19_c_build_dictionary_5select_candidates, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4tamp_19_c_build_dictionary_4select_candidates}; static PyObject *__pyx_pw_4tamp_19_c_build_dictionary_5select_candidates(PyObject *__pyx_self, #if CYTHON_METH_FASTCALL PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds #else PyObject *__pyx_args, PyObject *__pyx_kwds #endif ) { PyObject *__pyx_v_candidates = 0; PyObject *__pyx_v_multi_frag_content = 0; int __pyx_v_budget_remaining; int __pyx_v_overlap_threshold; #if !CYTHON_METH_FASTCALL CYTHON_UNUSED Py_ssize_t __pyx_nargs; #endif CYTHON_UNUSED PyObject *const *__pyx_kwvalues; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("select_candidates (wrapper)", 0); #if !CYTHON_METH_FASTCALL #if CYTHON_ASSUME_SAFE_SIZE __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); #else __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; #endif #endif __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); { PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_candidates,&__pyx_mstate_global->__pyx_n_u_multi_frag_content,&__pyx_mstate_global->__pyx_n_u_budget_remaining,&__pyx_mstate_global->__pyx_n_u_overlap_threshold,0}; PyObject* values[4] = {0,0,0,0}; const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 183, __pyx_L3_error) if (__pyx_kwds_len > 0) { switch (__pyx_nargs) { case 4: values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 183, __pyx_L3_error) CYTHON_FALLTHROUGH; case 3: values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 183, __pyx_L3_error) CYTHON_FALLTHROUGH; case 2: values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 183, __pyx_L3_error) CYTHON_FALLTHROUGH; case 1: values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 183, __pyx_L3_error) CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } const Py_ssize_t kwd_pos_args = __pyx_nargs; if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "select_candidates", 0) < (0)) __PYX_ERR(0, 183, __pyx_L3_error) for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("select_candidates", 1, 4, 4, i); __PYX_ERR(0, 183, __pyx_L3_error) } } } else if (unlikely(__pyx_nargs != 4)) { goto __pyx_L5_argtuple_error; } else { values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 183, __pyx_L3_error) values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 183, __pyx_L3_error) values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 183, __pyx_L3_error) values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 183, __pyx_L3_error) } __pyx_v_candidates = ((PyObject*)values[0]); __pyx_v_multi_frag_content = ((PyObject*)values[1]); __pyx_v_budget_remaining = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_budget_remaining == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 186, __pyx_L3_error) __pyx_v_overlap_threshold = __Pyx_PyLong_As_int(values[3]); if (unlikely((__pyx_v_overlap_threshold == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 187, __pyx_L3_error) } goto __pyx_L6_skip; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("select_candidates", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 183, __pyx_L3_error) __pyx_L6_skip:; goto __pyx_L4_argument_unpacking_done; __pyx_L3_error:; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } __Pyx_AddTraceback("tamp._c_build_dictionary.select_candidates", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_candidates), (&PyList_Type), 1, "candidates", 1))) __PYX_ERR(0, 184, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_multi_frag_content), (&PySet_Type), 1, "multi_frag_content", 1))) __PYX_ERR(0, 185, __pyx_L1_error) __pyx_r = __pyx_pf_4tamp_19_c_build_dictionary_4select_candidates(__pyx_self, __pyx_v_candidates, __pyx_v_multi_frag_content, __pyx_v_budget_remaining, __pyx_v_overlap_threshold); int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; /* function exit code */ goto __pyx_L0; __pyx_L1_error:; __pyx_r = NULL; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } goto __pyx_L7_cleaned_up; __pyx_L0:; for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { Py_XDECREF(values[__pyx_temp]); } __pyx_L7_cleaned_up:; __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_pf_4tamp_19_c_build_dictionary_4select_candidates(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_candidates, PyObject *__pyx_v_multi_frag_content, int __pyx_v_budget_remaining, int __pyx_v_overlap_threshold) { PyObject *__pyx_v_result = 0; PyObject *__pyx_v_accepted = 0; PyObject *__pyx_v_candidate = 0; int __pyx_v_i; int __pyx_v_k; int __pyx_v_n; int __pyx_v_used; int __pyx_v_write_idx; int __pyx_v_accepted_len; int __pyx_v_has_overlap; PyObject *__pyx_v_used_subs = 0; PyObject *__pyx_v_filtered = 0; PyObject *__pyx_r = NULL; /* … */ __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4tamp_19_c_build_dictionary_5select_candidates, 0, __pyx_mstate_global->__pyx_n_u_select_candidates, NULL, __pyx_mstate_global->__pyx_n_u_tamp__c_build_dictionary, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[2])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); #endif if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_select_candidates, __pyx_t_2) < (0)) __PYX_ERR(0, 183, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
184: list candidates,
185: set multi_frag_content,
186: int budget_remaining,
187: int overlap_threshold,
188: ):
189: """Extract all valid entries from candidates until budget is exhausted.
190:
191: Iterates through candidates (sorted by score descending). For each
192: valid candidate (exists in multi_frag_content, i.e. appears in 2+
193: fragments), accepts it and removes all candidates that share
194: content of >= overlap_threshold bytes. This prevents shifted
195: duplicates like "I DO NOT LIKE " and " DO NOT LIKE THE" from
196: both being selected.
197:
198: Returns a list of accepted entry bytes.
199: """
+200: cdef list result = []
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 200, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_result = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
201: cdef bytes accepted
202: cdef bytes candidate
203: cdef bytes sub
204: cdef int i, k, n, used
205: cdef int write_idx
206: cdef int accepted_len
207: cdef bint has_overlap
208:
209: # Track all overlap_threshold-length substrings of accepted entries.
+210: cdef set used_subs = set()
__pyx_t_1 = PySet_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 210, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_used_subs = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
211:
212: # Pre-filter: only keep candidates present in multi_frag_content.
+213: cdef list filtered = []
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 213, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_filtered = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+214: for i in range(len(candidates)):
if (unlikely(__pyx_v_candidates == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 214, __pyx_L1_error)
}
__pyx_t_2 = __Pyx_PyList_GET_SIZE(__pyx_v_candidates); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 214, __pyx_L1_error)
__pyx_t_3 = __pyx_t_2;
for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) {
__pyx_v_i = __pyx_t_4;
+215: candidate = <bytes>(<tuple>candidates[i])[0]
if (unlikely(__pyx_v_candidates == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 215, __pyx_L1_error)
}
if (unlikely(__Pyx_PyList_GET_ITEM(__pyx_v_candidates, __pyx_v_i) == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 215, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyTuple_GET_ITEM(((PyObject*)__Pyx_PyList_GET_ITEM(__pyx_v_candidates, __pyx_v_i)), 0);
__Pyx_INCREF(__pyx_t_1);
__Pyx_XDECREF_SET(__pyx_v_candidate, ((PyObject*)__pyx_t_1));
__pyx_t_1 = 0;
+216: if candidate in multi_frag_content:
if (unlikely(__pyx_v_multi_frag_content == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 216, __pyx_L1_error)
}
__pyx_t_5 = (__Pyx_PySet_ContainsTF(__pyx_v_candidate, __pyx_v_multi_frag_content, Py_EQ)); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 216, __pyx_L1_error)
if (__pyx_t_5) {
/* … */
}
}
+217: filtered.append(candidates[i])
if (unlikely(__pyx_v_candidates == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 217, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyList_GET_ITEM(__pyx_v_candidates, __pyx_v_i);
__Pyx_INCREF(__pyx_t_1);
__pyx_t_6 = __Pyx_PyList_Append(__pyx_v_filtered, __pyx_t_1); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 217, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
218:
+219: used = 0
__pyx_v_used = 0;
+220: n = len(filtered)
__pyx_t_2 = __Pyx_PyList_GET_SIZE(__pyx_v_filtered); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 220, __pyx_L1_error) __pyx_v_n = __pyx_t_2;
221:
+222: while n > 0 and used < budget_remaining:
while (1) {
__pyx_t_7 = (__pyx_v_n > 0);
if (__pyx_t_7) {
} else {
__pyx_t_5 = __pyx_t_7;
goto __pyx_L8_bool_binop_done;
}
__pyx_t_7 = (__pyx_v_used < __pyx_v_budget_remaining);
__pyx_t_5 = __pyx_t_7;
__pyx_L8_bool_binop_done:;
if (!__pyx_t_5) break;
+223: PyErr_CheckSignals()
__pyx_t_4 = PyErr_CheckSignals(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(0, 223, __pyx_L1_error)
224: # Find the first candidate that doesn't overlap with accepted entries.
+225: accepted = None
__Pyx_INCREF(Py_None); __Pyx_XDECREF_SET(__pyx_v_accepted, ((PyObject*)Py_None));
+226: for i in range(n):
__pyx_t_4 = __pyx_v_n;
__pyx_t_8 = __pyx_t_4;
for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_8; __pyx_t_9+=1) {
__pyx_v_i = __pyx_t_9;
+227: candidate = <bytes>(<tuple>filtered[i])[0]
if (unlikely(__Pyx_PyList_GET_ITEM(__pyx_v_filtered, __pyx_v_i) == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); __PYX_ERR(0, 227, __pyx_L1_error) } __pyx_t_1 = __Pyx_PyTuple_GET_ITEM(((PyObject*)__Pyx_PyList_GET_ITEM(__pyx_v_filtered, __pyx_v_i)), 0); __Pyx_INCREF(__pyx_t_1); __Pyx_XDECREF_SET(__pyx_v_candidate, ((PyObject*)__pyx_t_1)); __pyx_t_1 = 0;
+228: has_overlap = False
__pyx_v_has_overlap = 0;
+229: for k in range(len(candidate) - overlap_threshold + 1):
if (unlikely(__pyx_v_candidate == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 229, __pyx_L1_error)
}
__pyx_t_2 = __Pyx_PyBytes_GET_SIZE(__pyx_v_candidate); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 229, __pyx_L1_error)
__pyx_t_3 = ((__pyx_t_2 - __pyx_v_overlap_threshold) + 1);
__pyx_t_2 = __pyx_t_3;
for (__pyx_t_10 = 0; __pyx_t_10 < __pyx_t_2; __pyx_t_10+=1) {
__pyx_v_k = __pyx_t_10;
+230: if candidate[k : k + overlap_threshold] in used_subs:
if (unlikely(__pyx_v_candidate == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 230, __pyx_L1_error)
}
__pyx_t_1 = PySequence_GetSlice(__pyx_v_candidate, __pyx_v_k, (__pyx_v_k + __pyx_v_overlap_threshold)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 230, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_5 = (__Pyx_PySet_ContainsTF(__pyx_t_1, __pyx_v_used_subs, Py_EQ)); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 230, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
if (__pyx_t_5) {
/* … */
}
}
__pyx_L13_break:;
+231: has_overlap = True
__pyx_v_has_overlap = 1;
+232: break
goto __pyx_L13_break;
+233: if has_overlap:
if (__pyx_v_has_overlap) {
/* … */
}
+234: filtered[i] = None
if (unlikely((__Pyx_SetItemInt(__pyx_v_filtered, __pyx_v_i, Py_None, int, 1, __Pyx_PyLong_From_int, 1, 0, 0, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(0, 234, __pyx_L1_error)
+235: continue
goto __pyx_L10_continue;
+236: accepted = candidate
__Pyx_INCREF(__pyx_v_candidate); __Pyx_DECREF_SET(__pyx_v_accepted, __pyx_v_candidate);
+237: filtered[i] = None
if (unlikely((__Pyx_SetItemInt(__pyx_v_filtered, __pyx_v_i, Py_None, int, 1, __Pyx_PyLong_From_int, 1, 0, 0, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(0, 237, __pyx_L1_error)
+238: break
goto __pyx_L11_break;
__pyx_L10_continue:;
}
__pyx_L11_break:;
239:
+240: if accepted is None:
__pyx_t_5 = (__pyx_v_accepted == ((PyObject*)Py_None));
if (__pyx_t_5) {
/* … */
}
+241: break
goto __pyx_L7_break;
242:
+243: result.append(accepted)
__pyx_t_6 = __Pyx_PyList_Append(__pyx_v_result, __pyx_v_accepted); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 243, __pyx_L1_error)
+244: used += len(accepted)
if (unlikely(__pyx_v_accepted == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 244, __pyx_L1_error)
}
__pyx_t_3 = __Pyx_PyBytes_GET_SIZE(__pyx_v_accepted); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1))) __PYX_ERR(0, 244, __pyx_L1_error)
__pyx_v_used = (__pyx_v_used + __pyx_t_3);
+245: accepted_len = len(accepted)
if (unlikely(__pyx_v_accepted == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 245, __pyx_L1_error)
}
__pyx_t_3 = __Pyx_PyBytes_GET_SIZE(__pyx_v_accepted); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1))) __PYX_ERR(0, 245, __pyx_L1_error)
__pyx_v_accepted_len = __pyx_t_3;
246:
247: # Register all overlap_threshold-length substrings of the accepted entry.
+248: for k in range(accepted_len - overlap_threshold + 1):
__pyx_t_11 = ((__pyx_v_accepted_len - __pyx_v_overlap_threshold) + 1);
__pyx_t_12 = __pyx_t_11;
for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_12; __pyx_t_4+=1) {
__pyx_v_k = __pyx_t_4;
+249: used_subs.add(accepted[k : k + overlap_threshold])
if (unlikely(__pyx_v_accepted == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 249, __pyx_L1_error)
}
__pyx_t_1 = PySequence_GetSlice(__pyx_v_accepted, __pyx_v_k, (__pyx_v_k + __pyx_v_overlap_threshold)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 249, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_6 = PySet_Add(__pyx_v_used_subs, __pyx_t_1); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(0, 249, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
}
250:
251: # Compact the list (remove None entries).
+252: write_idx = 0
__pyx_v_write_idx = 0;
+253: for i in range(n):
__pyx_t_4 = __pyx_v_n;
__pyx_t_8 = __pyx_t_4;
for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_8; __pyx_t_9+=1) {
__pyx_v_i = __pyx_t_9;
+254: if filtered[i] is not None:
__pyx_t_5 = (__Pyx_PyList_GET_ITEM(__pyx_v_filtered, __pyx_v_i) != Py_None);
if (__pyx_t_5) {
/* … */
}
}
+255: filtered[write_idx] = filtered[i]
__pyx_t_1 = __Pyx_PyList_GET_ITEM(__pyx_v_filtered, __pyx_v_i); __Pyx_INCREF(__pyx_t_1); if (unlikely((__Pyx_SetItemInt(__pyx_v_filtered, __pyx_v_write_idx, __pyx_t_1, int, 1, __Pyx_PyLong_From_int, 1, 0, 0, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(0, 255, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+256: write_idx += 1
__pyx_v_write_idx = (__pyx_v_write_idx + 1);
+257: del filtered[write_idx:]
if (__Pyx_PyObject_DelSlice(__pyx_v_filtered, __pyx_v_write_idx, 0, NULL, NULL, NULL, 1, 0, 0) < (0)) __PYX_ERR(0, 257, __pyx_L1_error)
+258: n = write_idx
__pyx_v_n = __pyx_v_write_idx; } __pyx_L7_break:;
259:
+260: return result
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_result); __pyx_r = __pyx_v_result; goto __pyx_L0;