Disallowed Special Token about gpt-crawler HOT 2 OPEN

    encodeGenerator(lineToEncode, { allowedSpecial = new Set(), disallowedSpecial = new Set(), } = {}) {
        // Assuming ALL_SPECIAL_TOKENS is a placeholder for all special tokens
        if (disallowedSpecial.has(ALL_SPECIAL_TOKENS)) {
            disallowedSpecial = new Set(this.specialTokenMapping.keys());
            allowedSpecial.forEach(token => disallowedSpecial.delete(token));
        }
        
        // Check for disallowed tokens in the input
        disallowedSpecial.forEach(token => {
            if (lineToEncode.includes(token)) {
                throw new Error(`Disallowed special token found: ${token}`);
            }
        });
    
        return this.bytePairEncodingCoreProcessor.encodeNative(lineToEncode, allowedSpecial);
    }

export const EndOfText = "<EOT>";
export const FimPrefix = "<FimPrefix>";
export const FimMiddle = "<FimMiddle>";
export const FimSuffix = "<FimSuffix>";
export const ImStart = "<ImStart>";
export const ImEnd = "<ImEnd>";
export const ImSep = "<ImSep>";
export const EndOfPrompt = "<EndOfPrompt>";