module Uutf:sig
..end
Uutf
is a non-blocking streaming codec to decode and
encode the
UTF-8, UTF-16, UTF-16LE
and UTF-16BE encoding schemes. It can efficiently work character by
character without blocking on IO. Decoders perform
character position tracking and support newline normalization.
Functions are also provided to fold over the
characters of UTF encoded OCaml string values and to
directly encode characters in OCaml Buffer.t
values.
See examples of use.
v1.0.1 — homepage
val u_bom : Uchar.t
val u_rep : Uchar.t
typeencoding =
[ `UTF_16 | `UTF_16BE | `UTF_16LE | `UTF_8 ]
typedecoder_encoding =
[ `ISO_8859_1 | `US_ASCII | `UTF_16 | `UTF_16BE | `UTF_16LE | `UTF_8 ]
Uutf
. Unicode encoding
schemes plus US-ASCII and
ISO/IEC 8859-1 (latin-1).val encoding_of_string : string -> decoder_encoding option
val encoding_to_string : [< decoder_encoding ] -> string
typesrc =
[ `Channel of Pervasives.in_channel | `Manual | `String of string ]
`Manual
source the client
must provide input with Uutf.Manual.src
.typenln =
[ `ASCII of Uchar.t | `NLF of Uchar.t | `Readline of Uchar.t ]
`ASCII
, normalizes CR (U+000D
), LF (U+000A
) and CRLF
(<U+000D
, U+000A
>).`NLF
, normalizes the Unicode newline function (NLF). This is
NEL (U+0085
) and the normalizations of `ASCII
.`Readline
, normalizes for a Unicode readline function. This is FF
(U+000C
), LS (U+2028
), PS (U+2029
), and the normalizations
of `NLF
.`NLF
and
`Readline
normalizations allow to implement all the different
recommendations of Unicode's newline guidelines (section 5.8 in
Unicode 9.0.0).type
decoder
val decoder : ?nln:[< nln ] ->
?encoding:[< decoder_encoding ] -> [< src ] -> decoder
decoder nln encoding src
is a decoder that inputs from src
.
Byte order mark.
Byte order mark
(BOM) constraints are application dependent and prone to
misunderstandings (see the
FAQ). Hence,
Uutf
decoders have a simple rule: an initial BOM is always
removed from the input and not counted in character position
tracking. The function Uutf.decoder_removed_bom
does however return
true
if a BOM was removed so that all the information can be
recovered if needed.
For UTF-16BE and UTF-16LE the above rule is a violation of
conformance D96 and D97 of the standard. Uutf
favors the idea
that if there's a BOM, decoding with `UTF_16
or the `UTF_16XX
corresponding to the BOM should decode the same character sequence
(this is not the case if you stick to the standard). The client
can however regain conformance by consulting the result of
Uutf.decoder_removed_bom
and take appropriate action.
Encoding. encoding
specifies the decoded encoding
scheme. If `UTF_16
is used the endianness is determined
according to the standard: from a
BOM
if there is one, `UTF_16BE
otherwise.
If encoding
is unspecified it is guessed. The result of a guess
can only be `UTF_8
, `UTF_16BE
or `UTF_16LE
. The heuristic
looks at the first three bytes of input (or less if impossible)
and takes the first matching byte pattern in the table below.
xx = any byte
.. = any byte or no byte (input too small)
pp = positive byte
uu = valid UTF-8 first byte
Bytes | Guess | Rationale
---------+-----------+-----------------------------------------------
EF BB BF | `UTF_8 | UTF-8 BOM
FE FF .. | `UTF_16BE | UTF-16BE BOM
FF FE .. | `UTF_16LE | UTF-16LE BOM
00 pp .. | `UTF_16BE | ASCII UTF-16BE and U+0000 is often forbidden
pp 00 .. | `UTF_16LE | ASCII UTF-16LE and U+0000 is often forbidden
uu .. .. | `UTF_8 | ASCII UTF-8 or valid UTF-8 first byte.
xx xx .. | `UTF_16BE | Not UTF-8 => UTF-16, no BOM => UTF-16BE
.. .. .. | `UTF_8 | Single malformed UTF-8 byte or no input.
This heuristic is compatible both with BOM based
recognitition and
JSON-like encoding
recognition that relies on ASCII being present at the beginning
of the stream. Also, Uutf.decoder_removed_bom
will tell the client
if the guess was BOM based.
Newline normalization. If nln
is specified, the given
newline normalization is performed, see Uutf.nln
. Otherwise
all newlines are returned as found in the input.
Character position. The line number, column number, byte count
and character count of the last decoded character (including
`Malformed
ones) are respectively returned by Uutf.decoder_line
,
Uutf.decoder_col
, Uutf.decoder_byte_count
and Uutf.decoder_count
. Before
the first call to Uutf.decode
the line number is 1
and the column
is 0
. Each Uutf.decode
returning `Uchar
or `Malformed
increments the column until a newline. On a newline, the line
number is incremented and the column set to zero. For example the
line is 2
and column 0
after the first newline was
decoded. This can be understood as if Uutf.decode
was moving an
insertion point to the right in the data. A newline is
anything normalized by `Readline
, see Uutf.nln
.
Uutf
assumes that each Unicode scalar value has a column width
of 1. The same assumption may not be made by the display program
(e.g. for emacs
' compilation mode you need to set
compilation-error-screen-columns
to nil
). The problem is in
general difficult to solve without interaction or convention with the
display program's rendering engine. Depending on the context better column
increments can be implemented by using Uucp.Break.tty_width_hint
or
grapheme cluster boundaries (see Uuseg
).
val decode : decoder -> [ `Await | `End | `Malformed of string | `Uchar of Uchar.t ]
decode d
is:
`Await
if d
has a `Manual
input source and awaits
for more input. The client must use Uutf.Manual.src
to provide it.`Uchar u
if a Unicode scalar value u
was decoded.`End
if the end of input was reached.`Malformed bytes
if the bytes
sequence is malformed according to
the decoded encoding scheme. If you are interested in a best-effort
decoding you can still continue to decode after an error until the
decoder synchronizes again on valid bytes. It may however be a good
idea to signal the malformed characters by adding an Uutf.u_rep
character to the parsed data, see the examples.
Note. Repeated invocation always eventually returns `End
, even
in case of errors.
val decoder_encoding : decoder -> decoder_encoding
decoder_encoding d
is d
's the decoded encoding scheme of d
.
Warning. If the decoder guesses the encoding or uses `UTF_16
,
rely on this value only after the first `Uchar
was decoded.
val decoder_line : decoder -> int
decoder_line d
is the line number of the last
decoded (or malformed) character. See Uutf.decoder
for details.val decoder_col : decoder -> int
decoder_col d
is the column number of the last decoded
(or malformed) character. See Uutf.decoder
for details.val decoder_byte_count : decoder -> int
decoder_byte_count d
is the number of bytes already decoded on
d
(including malformed ones). This is the last Uutf.decode
's
end byte offset counting from the beginning of the stream.val decoder_count : decoder -> int
decoder_count d
is the number of characters already decoded on d
(including malformed ones). See Uutf.decoder
for details.val decoder_removed_bom : decoder -> bool
decoder_removed_bom d
is true
iff an initial
BOM was
removed from the input stream. See Uutf.decoder
for details.val decoder_src : decoder -> src
decoder_src d
is d
's input source.val decoder_nln : decoder -> nln option
decoder_nln d
returns d
's newline normalization (if any).val pp_decode : Format.formatter ->
[< `Await | `End | `Malformed of string | `Uchar of Uchar.t ] -> unit
pp_decode ppf v
prints an unspecified representation of v
on
ppf
.typedst =
[ `Buffer of Buffer.t | `Channel of Pervasives.out_channel | `Manual ]
`Manual
destination the client
must provide output storage with Uutf.Manual.dst
.type
encoder
val encoder : [< encoding ] -> [< dst ] -> encoder
encoder encoding dst
is an encoder for encoding
that outputs
to dst
.
Note. No initial
BOM
is encoded. If needed, this duty is left to the client.
val encode : encoder -> [< `Await | `End | `Uchar of Uchar.t ] -> [ `Ok | `Partial ]
encode e v
is :
`Partial
iff e
has a `Manual
destination and needs more output
storage. The client must use Uutf.Manual.dst
to provide a new buffer
and then call Uutf.encode
with `Await
until `Ok
is returned.`Ok
when the encoder is ready to encode a new `Uchar
or `End
For `Manual
destination, encoding `End
always returns
`Partial
, the client should continue as usual with `Await
until `Ok
is returned at which point Uutf.Manual.dst_rem
e
is
guaranteed to be the size of the last provided buffer (i.e. nothing
was written).
Raises. Invalid_argument
if an `Uchar
or `End
is encoded
after a `Partial
encode.
val encoder_encoding : encoder -> encoding
encoder_encoding e
is e
's encoding.val encoder_dst : encoder -> dst
encoder_dst e
is e
's output destination.module Manual:sig
..end
module String:sig
..end
string
values.
module Buffer:sig
..end
Buffer.t
values.
The value of lines src
is the list of lines in src
as UTF-8
encoded OCaml strings. Line breaks are determined according to the
recommendation R4 for a readline
function in section 5.8 of
Unicode 9.0.0. If a decoding error occurs we silently replace the
malformed sequence by the replacement character Uutf.u_rep
and continue.
let lines ?encoding (src : [`Channel of in_channel | `String of string]) =
let rec loop d buf acc = match Uutf.decode d with
| `Uchar u ->
begin match Uchar.to_int u with
| 0x000A ->
let line = Buffer.contents buf in
Buffer.clear buf; loop d buf (line :: acc)
| _ ->
Uutf.Buffer.add_utf_8 buf u; loop d buf acc
end
| `End -> List.rev (Buffer.contents buf :: acc)
| `Malformed _ -> Uutf.Buffer.add_utf_8 buf Uutf.u_rep; loop d buf acc
| `Await -> assert false
in
let nln = `Readline (Uchar.of_int 0x000A) in
loop (Uutf.decoder ~nln ?encoding src) (Buffer.create 512) []
Using the `Manual
interface, lines_fd
does the same but on a Unix file
descriptor.
let lines_fd ?encoding (fd : Unix.file_descr) =
let rec loop fd s d buf acc = match Uutf.decode d with
| `Uchar u ->
begin match Uchar.to_int u with
| 0x000A ->
let line = Buffer.contents buf in
Buffer.clear buf; loop fd s d buf (line :: acc)
| _ ->
Uutf.Buffer.add_utf_8 buf u; loop fd s d buf acc
end
| `End -> List.rev (Buffer.contents buf :: acc)
| `Malformed _ -> Uutf.Buffer.add_utf_8 buf Uutf.u_rep; loop fd s d buf acc
| `Await ->
let rec unix_read fd s j l = try Unix.read fd s j l with
| Unix.Unix_error (Unix.EINTR, _, _) -> unix_read fd s j l
in
let rc = unix_read fd s 0 (Bytes.length s) in
Uutf.Manual.src d s 0 rc; loop fd s d buf acc
in
let s = Bytes.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
let nln = `Readline (Uchar.of_int 0x000A) in
loop fd s (Uutf.decoder ~nln ?encoding `Manual) (Buffer.create 512) []
The result of recode src out_encoding dst
has the characters of
src
written on dst
with encoding out_encoding
. If a
decoding error occurs we silently replace the malformed sequence
by the replacement character Uutf.u_rep
and continue. Note that we
don't add an initial
BOM to dst
,
recoding will thus loose the initial BOM src
may have. Whether
this is a problem or not depends on the context.
let recode ?nln ?encoding out_encoding
(src : [`Channel of in_channel | `String of string])
(dst : [`Channel of out_channel | `Buffer of Buffer.t])
=
let rec loop d e = match Uutf.decode d with
| `Uchar _ as u -> ignore (Uutf.encode e u); loop d e
| `End -> ignore (Uutf.encode e `End)
| `Malformed _ -> ignore (Uutf.encode e (`Uchar Uutf.u_rep)); loop d e
| `Await -> assert false
in
let d = Uutf.decoder ?nln ?encoding src in
let e = Uutf.encoder out_encoding dst in
loop d e
Using the `Manual
interface, recode_fd
does the same but between
Unix file descriptors.
let recode_fd ?nln ?encoding out_encoding
(fdi : Unix.file_descr)
(fdo : Unix.file_descr)
=
let rec encode fd s e v = match Uutf.encode e v with `Ok -> ()
| `Partial ->
let rec unix_write fd s j l =
let rec write fd s j l = try Unix.single_write fd s j l with
| Unix.Unix_error (Unix.EINTR, _, _) -> write fd s j l
in
let wc = write fd s j l in
if wc < l then unix_write fd s (j + wc) (l - wc) else ()
in
unix_write fd s 0 (Bytes.length s - Uutf.Manual.dst_rem e);
Uutf.Manual.dst e s 0 (Bytes.length s);
encode fd s e `Await
in
let rec loop fdi fdo ds es d e = match Uutf.decode d with
| `Uchar _ as u -> encode fdo es e u; loop fdi fdo ds es d e
| `End -> encode fdo es e `End
| `Malformed _ -> encode fdo es e (`Uchar Uutf.u_rep); loop fdi fdo ds es d e
| `Await ->
let rec unix_read fd s j l = try Unix.read fd s j l with
| Unix.Unix_error (Unix.EINTR, _, _) -> unix_read fd s j l
in
let rc = unix_read fdi ds 0 (Bytes.length ds) in
Uutf.Manual.src d ds 0 rc; loop fdi fdo ds es d e
in
let ds = Bytes.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
let es = Bytes.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
let d = Uutf.decoder ?nln ?encoding `Manual in
let e = Uutf.encoder out_encoding `Manual in
Uutf.Manual.dst e es 0 (Bytes.length es);
loop fdi fdo ds es d e