diff --git a/core-foundation-sys/src/lib.rs b/core-foundation-sys/src/lib.rs index c8468c2..16df21a 100644 --- a/core-foundation-sys/src/lib.rs +++ b/core-foundation-sys/src/lib.rs @@ -44,6 +44,7 @@ pub mod set; pub mod socket; pub mod stream; pub mod string; +pub mod string_tokenizer; pub mod timezone; pub mod url; #[cfg(target_os = "macos")] diff --git a/core-foundation-sys/src/string_tokenizer.rs b/core-foundation-sys/src/string_tokenizer.rs new file mode 100644 index 0000000..f4c23d4 --- /dev/null +++ b/core-foundation-sys/src/string_tokenizer.rs @@ -0,0 +1,65 @@ +// Copyright 2023 The Servo Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::os::raw::c_void; + +use base::{CFAllocatorRef, CFTypeID, CFRange, CFIndex, CFOptionFlags, CFTypeRef}; +use string::CFStringRef; +use locale::CFLocaleRef; +use array::CFMutableArrayRef; + +#[repr(C)] +pub struct __CFStringTokenizer(c_void); +pub type CFStringTokenizerRef = *mut __CFStringTokenizer; + +pub type CFStringTokenizerTokenType = CFOptionFlags; + +pub const kCFStringTokenizerTokenNone: CFStringTokenizerTokenType = 0; +pub const kCFStringTokenizerTokenNormal: CFStringTokenizerTokenType = 1 << 0; +pub const kCFStringTokenizerTokenHasSubTokensMask: CFStringTokenizerTokenType = 1 << 1; +pub const kCFStringTokenizerTokenHasDerivedSubTokensMask: CFStringTokenizerTokenType = 1 << 2; +pub const kCFStringTokenizerTokenHasHasNumbersMask: CFStringTokenizerTokenType = 1 << 3; +pub const kCFStringTokenizerTokenHasNonLettersMask: CFStringTokenizerTokenType = 1 << 4; +pub const kCFStringTokenizerTokenIsCJWordMask: CFStringTokenizerTokenType = 1 << 5; + +/* Tokenization Modifiers */ +pub const kCFStringTokenizerUnitWord: CFOptionFlags = 0; +pub const kCFStringTokenizerUnitSentence: CFOptionFlags = 1; +pub const kCFStringTokenizerUnitParagraph: CFOptionFlags = 2; +pub const kCFStringTokenizerUnitLineBreak: CFOptionFlags = 3; +pub const kCFStringTokenizerUnitWordBoundary: CFOptionFlags = 4; +pub const kCFStringTokenizerAttributeLatinTranscription: CFOptionFlags = 1 << 16; +pub const kCFStringTokenizerAttributeLanguage: CFOptionFlags = 1 << 17; + +extern { + /* + * CFStringTokenizer.h + */ + + /* Creating a Tokenizer */ + pub fn CFStringTokenizerCreate(alloc: CFAllocatorRef, string: CFStringRef, range: CFRange, options: CFOptionFlags, locale: CFLocaleRef) -> CFStringTokenizerRef; + + /* Setting the String */ + pub fn CFStringTokenizerSetString(tokenizer: CFStringTokenizerRef, string: CFStringRef, range: CFRange); + + /* Changing the Location */ + pub fn CFStringTokenizerAdvanceToNextToken(tokenizer: CFStringTokenizerRef) -> CFStringTokenizerTokenType; + pub fn CFStringTokenizerGoToTokenAtIndex(tokenizer: CFStringTokenizerRef, index: CFIndex) -> CFStringTokenizerTokenType; + + /* Getting Information About the Current Token */ + pub fn CFStringTokenizerCopyCurrentTokenAttribute(tokenizer: CFStringTokenizerRef, attribute: CFOptionFlags) -> CFTypeRef; + pub fn CFStringTokenizerGetCurrentTokenRange(tokenizer: CFStringTokenizerRef) -> CFRange; + pub fn CFStringTokenizerGetCurrentSubTokens(tokenizer: CFStringTokenizerRef, ranges: *mut CFRange, maxRangeLength: CFIndex, derivedSubTokens: CFMutableArrayRef) -> CFIndex; + + /* Identifying a Language */ + pub fn CFStringTokenizerCopyBestStringLanguage(string: CFStringRef, range: CFRange) -> CFStringRef; + + /* Getting the CFStringTokenizer Type ID */ + pub fn CFStringTokenizerGetTypeID() -> CFTypeID; +}