|
9
|
1 /*
|
|
|
2 ** Copyright (c) 2014-2017, Eren Okka
|
|
|
3 **
|
|
|
4 ** This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
5 ** License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
6 ** file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
|
7 */
|
|
|
8
|
|
|
9 #pragma once
|
|
|
10
|
|
|
11 #include "element.h"
|
|
|
12 #include "options.h"
|
|
|
13 #include "string.h"
|
|
|
14 #include "token.h"
|
|
|
15
|
|
|
16 namespace anitomy {
|
|
|
17
|
|
|
18 class Tokenizer {
|
|
|
19 public:
|
|
|
20 Tokenizer(const string_t& filename, Elements& elements, const Options& options, token_container_t& tokens);
|
|
|
21
|
|
|
22 Tokenizer(const Tokenizer&) = delete;
|
|
|
23 Tokenizer& operator=(const Tokenizer&) = delete;
|
|
|
24
|
|
|
25 bool Tokenize();
|
|
|
26
|
|
|
27 private:
|
|
|
28 void AddToken(TokenCategory category, bool enclosed, const TokenRange& range);
|
|
|
29 void TokenizeByBrackets();
|
|
|
30 void TokenizeByPreidentified(bool enclosed, const TokenRange& range);
|
|
|
31 void TokenizeByDelimiters(bool enclosed, const TokenRange& range);
|
|
|
32
|
|
|
33 string_t GetDelimiters(const TokenRange& range) const;
|
|
|
34 void ValidateDelimiterTokens();
|
|
|
35
|
|
|
36 Elements& elements_;
|
|
|
37 const string_t& filename_;
|
|
|
38 const Options& options_;
|
|
|
39 token_container_t& tokens_;
|
|
|
40 };
|
|
|
41
|
|
|
42 } // namespace anitomy
|
