/*
 *  HTML Entity & Encoding normalization.
 *
 *  Copyright (C) 2013-2022 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
 *  Copyright (C) 2007-2013 Sourcefire, Inc.
 *
 *  Authors: Török Edvin
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License version 2 as
 *  published by the Free Software Foundation.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 *  MA 02110-1301, USA.
 */

#ifndef _ENTITIES_H
#define _ENTITIES_H

#include "clamav-config.h"

#ifdef HAVE_ICONV
#include <iconv.h>
#endif

#include "clamav-types.h"
#include "hashtab.h"
#include "htmlnorm.h"

// clang-format off
#define UCS4_1234           "UCS-4BE"
#define UCS4_4321           "UCS-4LE"
#define UCS4_2143           "UCS4"
#define UCS4_3412           "UCS-4"
#define UTF16_BE            "UTF-16BE"
#define UTF16_LE            "UTF-16LE"
#define UTF8                "UTF-8"
#define UNDECIDED_32_1234   UCS4_1234
#define UNDECIDED_32_4321   UCS4_4321
#define UNDECIDED_32_2143   UCS4_2143
#define UNDECIDED_32_3412   UCS4_3412
#define UNDECIDED_16_BE     UTF16_BE
#define UNDECIDED_16_LE     UTF16_LE
#define UNDECIDED_8         "ISO-8859-1"

#define CODEPAGE_JAPANESE_SHIFT_JIS 932     /* Japanese Shift-JIS */
#define CODEPAGE_KOREAN_UNIFIED     949     /* Korean Unified Code */
#define CODEPAGE_UTF16_LE           1200    /* UTF16 Little Endian */
#define CODEPAGE_UTF16_BE           1201    /* UTF16 Big Endian */
#define CODEPAGE_US_7BIT_ASCII      20127   /* US-ASCII (7-bit)  */
#define CODEPAGE_ISO8859_1          28591   /* ISO 8859-1 Latin 1; Western European (ISO) */
#define CODEPAGE_UTF8               65001   /* UTF-8 */
// clang-format on

/* string conversion */
struct codepage_entry {
    uint16_t codepage;
    const char* encoding;
};

#define NUMCODEPAGES (sizeof(codepage_entries) / sizeof(struct codepage_entry))
/* MAINTAIN - the array in codepage value sorted order */
static const struct codepage_entry codepage_entries[] = {
    {37, "IBM037"},           /* IBM EBCDIC US-Canada */
    {437, "IBM437"},          /* OEM United States */
    {500, "IBM500"},          /* IBM EBCDIC International */
    {708, "ASMO-708"},        /* Arabic (ASMO 708) */
    {709, NULL},              /* Arabic (ASMO-449+, BCON V4) */
    {710, NULL},              /* Arabic - Transparent Arabic */
    {720, NULL},              /* Arabic (Transparent ASMO); Arabic (DOS) */
    {737, NULL},              /* OEM Greek (formerly 437G); Greek (DOS) */
    {775, "IBM775"},          /* OEM Baltic; Baltic (DOS) */
    {850, "IBM850"},          /* OEM Multilingual Latin 1; Western European (DOS) */
    {852, "IBM852"},          /* OEM Latin 2; Central European (DOS) */
    {855, "IBM855"},          /* OEM Cyrillic (primarily Russian) */
    {857, "IBM857"},          /* OEM Turkish; Turkish (DOS) */
    {858, NULL},              /* OEM Multilingual Latin 1 + Euro symbol */
    {860, "IBM860"},          /* OEM Portuguese; Portuguese (DOS) */
    {861, "IBM861"},          /* OEM Icelandic; Icelandic (DOS) */
    {862, NULL},              /* OEM Hebrew; Hebrew (DOS) */
    {863, "IBM863"},          /* OEM French Canadian; French Canadian (DOS) */
    {864, "IBM864"},          /* OEM Arabic; Arabic (864) */
    {865, "IBM865"},          /* OEM Nordic; Nordic (DOS) */
    {866, "CP866"},           /* OEM Russian; Cyrillic (DOS) */
    {869, "IBM869"},          /* OEM Modern Greek; Greek, Modern (DOS) */
    {870, "IBM870"},          /* IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2 */
    {874, "WINDOWS-874"},     /* ANSI/OEM Thai (ISO 8859-11); Thai (Windows) */
    {875, "CP875"},           /* IBM EBCDIC Greek Modern */
    {932, "SHIFT_JIS"},       /* ANSI/OEM Japanese; Japanese (Shift-JIS) */
    {936, "GB2312"},          /* ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) */
    {949, "CP949"},           /* ANSI/OEM Korean (Unified Hangul Code) */
    {950, "BIG5"},            /* ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) */
    {1026, "IBM1026"},        /* IBM EBCDIC Turkish (Latin 5) */
    {1047, NULL},             /* IBM EBCDIC Latin 1/Open System */
    {1140, NULL},             /* IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro) */
    {1141, NULL},             /* IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro) */
    {1142, NULL},             /* IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro) */
    {1143, NULL},             /* IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro) */
    {1144, NULL},             /* IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro) */
    {1145, NULL},             /* IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro) */
    {1146, NULL},             /* IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro) */
    {1147, NULL},             /* IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro) */
    {1148, NULL},             /* IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro) */
    {1149, NULL},             /* IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro) */
    {1200, "UTF-16LE"},       /* Unicode UTF-16, little endian byte order (BMP of ISO 10646); available only to managed applications */
    {1201, "UTF-16BE"},       /* Unicode UTF-16, big endian byte order; available only to managed applications */
    {1250, "WINDOWS-1250"},   /* ANSI Central European; Central European (Windows) */
    {1251, "WINDOWS-1251"},   /* ANSI Cyrillic; Cyrillic (Windows) */
    {1252, "WINDOWS-1252"},   /* ANSI Latin 1; Western European (Windows) */
    {1253, "WINDOWS-1253"},   /* ANSI Greek; Greek (Windows) */
    {1254, "WINDOWS-1254"},   /* ANSI Turkish; Turkish (Windows) */
    {1255, "WINDOWS-1255"},   /* ANSI Hebrew; Hebrew (Windows) */
    {1256, "WINDOWS-1256"},   /* ANSI Arabic; Arabic (Windows) */
    {1257, "WINDOWS-1257"},   /* ANSI Baltic; Baltic (Windows) */
    {1258, "WINDOWS-1258"},   /* ANSI/OEM Vietnamese; Vietnamese (Windows) */
    {1361, "JOHAB"},          /* Korean (Johab) */
    {10000, "MACINTOSH"},     /* MAC Roman; Western European (Mac) */
    {10001, NULL},            /* Japanese (Mac) */
    {10002, NULL},            /* MAC Traditional Chinese (Big5); Chinese Traditional (Mac) */
    {10003, NULL},            /* Korean (Mac) */
    {10004, NULL},            /* Arabic (Mac) */
    {10005, NULL},            /* Hebrew (Mac) */
    {10006, NULL},            /* Greek (Mac) */
    {10007, NULL},            /* Cyrillic (Mac) */
    {10008, NULL},            /* MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac) */
    {10010, NULL},            /* Romanian (Mac) */
    {10017, NULL},            /* Ukrainian (Mac) */
    {10021, NULL},            /* Thai (Mac) */
    {10029, NULL},            /* MAC Latin 2; Central European (Mac) */
    {10079, NULL},            /* Icelandic (Mac) */
    {10081, NULL},            /* Turkish (Mac) */
    {10082, NULL},            /* Croatian (Mac) */
    {12000, "UTF-32LE"},      /* Unicode UTF-32, little endian byte order; available only to managed applications */
    {12001, "UTF-32BE"},      /* Unicode UTF-32, big endian byte order; available only to managed applications */
    {20000, NULL},            /* CNS Taiwan; Chinese Traditional (CNS) */
    {20001, NULL},            /* TCA Taiwan */
    {20002, NULL},            /* Eten Taiwan; Chinese Traditional (Eten) */
    {20003, NULL},            /* IBM5550 Taiwan */
    {20004, NULL},            /* TeleText Taiwan */
    {20005, NULL},            /* Wang Taiwan */
    {20105, NULL},            /* IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5) */
    {20106, NULL},            /* IA5 German (7-bit) */
    {20107, NULL},            /* IA5 Swedish (7-bit) */
    {20108, NULL},            /* IA5 Norwegian (7-bit) */
    {20127, "US-ASCII"},      /* US-ASCII (7-bit) */
    {20261, NULL},            /* T.61 */
    {20269, NULL},            /* ISO 6937 Non-Spacing Accent */
    {20273, "IBM273"},        /* IBM EBCDIC Germany */
    {20277, "IBM277"},        /* IBM EBCDIC Denmark-Norway */
    {20278, "IBM278"},        /* IBM EBCDIC Finland-Sweden */
    {20280, "IBM280"},        /* IBM EBCDIC Italy */
    {20284, "IBM284"},        /* IBM EBCDIC Latin America-Spain */
    {20285, "IBM285"},        /* IBM EBCDIC United Kingdom */
    {20290, "IBM290"},        /* IBM EBCDIC Japanese Katakana Extended */
    {20297, "IBM297"},        /* IBM EBCDIC France */
    {20420, "IBM420"},        /* IBM EBCDIC Arabic */
    {20423, "IBM423"},        /* IBM EBCDIC Greek */
    {20424, "IBM424"},        /* IBM EBCDIC Hebrew */
    {20833, NULL},            /* IBM EBCDIC Korean Extended */
    {20838, NULL},            /* IBM EBCDIC Thai */
    {20866, "KOI8-R"},        /* Russian (KOI8-R); Cyrillic (KOI8-R) */
    {20871, "IBM871"},        /* IBM EBCDIC Icelandic */
    {20880, "IBM880"},        /* IBM EBCDIC Cyrillic Russian */
    {20905, "IBM905"},        /* IBM EBCDIC Turkish */
    {20924, NULL},            /* IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) */
    {20932, "EUC-JP"},        /* Japanese (JIS 0208-1990 and 0212-1990) */
    {20936, NULL},            /* Simplified Chinese (GB2312); Chinese Simplified (GB2312-80) */
    {20949, NULL},            /* Korean Wansung */
    {21025, "CP1025"},        /* IBM EBCDIC Cyrillic Serbian-Bulgarian */
    {21027, NULL},            /* (deprecated) */
    {21866, "KOI8-U"},        /* Ukrainian (KOI8-U); Cyrillic (KOI8-U) */
    {28591, "ISO-8859-1"},    /* ISO 8859-1 Latin 1; Western European (ISO) */
    {28592, "ISO-8859-2"},    /* ISO 8859-2 Central European; Central European (ISO) */
    {28593, "ISO-8859-3"},    /* ISO 8859-3 Latin 3 */
    {28594, "ISO-8859-4"},    /* ISO 8859-4 Baltic */
    {28595, "ISO-8859-5"},    /* ISO 8859-5 Cyrillic */
    {28596, "ISO-8859-6"},    /* ISO 8859-6 Arabic */
    {28597, "ISO-8859-7"},    /* ISO 8859-7 Greek */
    {28598, "ISO-8859-8"},    /* ISO 8859-8 Hebrew; Hebrew (ISO-Visual) */
    {28599, "ISO-8859-9"},    /* ISO 8859-9 Turkish */
    {28603, "ISO-8859-13"},   /* ISO 8859-13 Estonian */
    {28605, "ISO-8859-15"},   /* ISO 8859-15 Latin 9 */
    {29001, NULL},            /* Europa 3 */
    {38598, NULL},            /* ISO 8859-8 Hebrew; Hebrew (ISO-Logical) */
    {50220, "ISO-2022-JP"},   /* ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS) (guess) */
    {50221, "ISO-2022-JP-2"}, /* ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana) (guess) */
    {50222, "ISO-2022-JP-3"}, /* ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI) (guess) */
    {50225, "ISO-2022-KR"},   /* ISO 2022 Korean */
    {50227, NULL},            /* ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022) */
    {50229, NULL},            /* ISO 2022 Traditional Chinese */
    {50930, NULL},            /* EBCDIC Japanese (Katakana) Extended */
    {50931, NULL},            /* EBCDIC US-Canada and Japanese */
    {50933, NULL},            /* EBCDIC Korean Extended and Korean */
    {50935, NULL},            /* EBCDIC Simplified Chinese Extended and Simplified Chinese */
    {50936, NULL},            /* EBCDIC Simplified Chinese */
    {50937, NULL},            /* EBCDIC US-Canada and Traditional Chinese */
    {50939, NULL},            /* EBCDIC Japanese (Latin) Extended and Japanese */
    {51932, "EUC-JP"},        /* EUC Japanese */
    {51936, "EUC-CN"},        /* EUC Simplified Chinese; Chinese Simplified (EUC) */
    {51949, "EUC-KR"},        /* EUC Korean */
    {51950, NULL},            /* EUC Traditional Chinese */
    {52936, NULL},            /* HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ) */
    {54936, "GB18030"},       /* Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030) */
    {57002, NULL},            /* ISCII Devanagari */
    {57003, NULL},            /* ISCII Bengali */
    {57004, NULL},            /* ISCII Tamil */
    {57005, NULL},            /* ISCII Telugu */
    {57006, NULL},            /* ISCII Assamese */
    {57007, NULL},            /* ISCII Oriya */
    {57008, NULL},            /* ISCII Kannada */
    {57009, NULL},            /* ISCII Malayalam */
    {57010, NULL},            /* ISCII Gujarati */
    {57011, NULL},            /* ISCII Punjabi */
    {65000, "UTF-7"},         /* Unicode (UTF-7) */
    {65001, "UTF-8"}          /* Unicode (UTF-8) */
};

#define MAX_ENTITY_SIZE 22

struct entity_conv {
    unsigned char entity_buff[MAX_ENTITY_SIZE + 2];
};

typedef enum encodings {
    E_UCS4,
    E_UTF16,
    E_UCS4_1234,
    E_UCS4_4321,
    E_UCS4_2143,
    E_UCS4_3412,
    E_UTF16_BE,
    E_UTF16_LE,
    E_UTF8,
    E_UNKNOWN,
    E_OTHER
} encoding_t;

unsigned char* u16_normalize_tobuffer(uint16_t u16, unsigned char* dst, size_t dst_size);
const char* entity_norm(struct entity_conv* conv, const unsigned char* entity);
const char* encoding_detect_bom(const unsigned char* bom, const size_t length);
int encoding_normalize_toascii(const m_area_t* in_m_area, const char* initial_encoding, m_area_t* out_m_area);

/**
 * @brief Convert string to UTF-8, given Windows codepage.
 *
 * @param in                string buffer
 * @param in_size           length of string buffer in bytes
 * @param codepage          Windows code page https://docs.microsoft.com/en-us/windows/desktop/Intl/code-page-identifiers)
 * @param [out] out         pointer to receive malloc'ed utf-8 buffer.
 * @param [out] out_size    pointer to receive size of utf-8 buffer, not including null terminating character.
 * @return cl_error_t   CL_SUCCESS if success. CL_BREAK if unable to because iconv is unavailable.  Other error code if outright failure.
 */
cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, char** out, size_t* out_size);

char* cli_utf16toascii(const char* str, unsigned int length);

char* cli_utf16_to_utf8(const char* utf16, size_t length, encoding_t type);

int cli_isutf8(const char* buf, unsigned int len);

#endif
