Username404-59
/
Yerbacon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a "SEPARATOR" token type

This commit is contained in:
Username404 2021-09-10 22:15:05 +02:00
parent 37a6c0e047
commit 6de05633d9
Signed by: Username404-59
GPG Key ID: 7AB361FBB257A5D1
5 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/etc/lexer.cpp
View File

@ -9,7 +9,7 @@ tok::type getIdentifierCharType(const char& Char) {
else return UNEXPECTED;
}
vector<tok> lex(const string& in)
vector<tok> lex(const string& in, const char separatorCharacter)
{
vector<tok> resVal;
unsigned long lineNumber = 1;
@ -21,9 +21,9 @@ vector<tok> lex(const string& in)
case LBRACE: case RBRACE: case LBRACKET: case RBRACKET:
case PLUS: case HYPHEN: case LCOMP: case RCOMP:
case DOT: case DOLLAR_SIGN: case SQUOTE:
resVal.emplace_back(static_cast<tok::type>(current), string(1, current), lineNumber);
resVal.emplace_back(static_cast<tok::type>(current), string(1, current), lineNumber); break;
[[likely]] case '\n': ++lineNumber; case ';': resVal.emplace_back(SEPARATOR, string(1, separatorCharacter), lineNumber);
[[likely]] case ' ': case '\t': case '\r': break;
[[likely]] case '\n': ++lineNumber; break;
default: {
tok::type type = getIdentifierCharType(current);
bool isTypeString = (type == STRING);
@ -32,8 +32,8 @@ vector<tok> lex(const string& in)
case IDENTIFIER: case NUMBER: {
string formedString;
for (;i < in.size(); ++i) {
tok::type currentCharType = getIdentifierCharType(in[i]);
bool isString = currentCharType == STRING;
const tok::type currentCharType = getIdentifierCharType(in[i]);
const bool isString = currentCharType == STRING;
if (i == in.size() - 1 && not isString) throw tok::LexerException("A never ending string was found", lineNumber);
if ((currentCharType == type || isTypeString) && !isString) {
formedString += string(1, in[i]);

6
src/headers/lex.hpp
View File

@ -11,7 +11,7 @@ struct tok {
using Yerbacon::Exception::Exception;
};
enum type: const unsigned short {
UNEXPECTED = std::numeric_limits<unsigned char>::max() + 1, IDENTIFIER, NUMBER, ALPHACHAR,
UNEXPECTED = std::numeric_limits<unsigned char>::max() + 1, IDENTIFIER, NUMBER, ALPHACHAR, SEPARATOR,
DEFINE = '=', TAG = '#', DOLLAR_SIGN = '$', DOT = '.',
PLUS = '+', LPAR = '(', LBRACE = '{', LBRACKET = '[', RPAR = ')',
RBRACE = '}', RBRACKET = ']',
@ -22,9 +22,9 @@ struct tok {
const type toktype;
const std::string toktext;
const unsigned long line = 0;
tok(type Type, std::string_view Text, auto line): toktype(Type), toktext(Text), line(line) {}
tok(type Type, std::string_view Text, decltype(line) line): toktype(Type), toktext(Text), line(line) {}
friend std::ostream& operator<<(std::ostream& output, const tok& it) { return output << it.toktext; }
};
std::vector<tok> lex(const std::string& in);
std::vector<tok> lex(const std::string& in, char separatorCharacter);
#endif //YERBACON_TEST_H

2
src/headers/misc.hpp
View File

@ -6,6 +6,6 @@ void outputFileContent(const string& file, string_view content);
#include "lex.hpp"
#include "parsing/Parser.hpp"
inline auto parseString(const string& toParse) { return Parser::parseVector(lex(toParse)); }
inline auto parseString(const string& toParse, const char lineSeparator) { return Parser::parseVector(lex(toParse, lineSeparator)); }
#endif //YERBACON_MISC_HPP

2
src/headers/transpiler/Target.hpp
View File

@ -100,7 +100,7 @@ shared_ptr<Target> Target::forName(string_view name) {
}
}
shared_ptr<Target> target;
#define ADDTARGET(X) target = make_shared<X>(X());
#define ADDTARGET(X) target = shared_ptr<X>(new (X));
switch (selected) {
#ifdef LUA_HPP
case LUA: ADDTARGET(LUA_HPP); break;

9
src/main.cpp
View File

@ -14,6 +14,7 @@ int main(int argc, char* argv[]) {
string target = ".lua";
bool printResult = false;
bool parallel = false;
bool newLinesSeparator = true;
if (argc > 0) {
vector<string_view> files;
@ -22,15 +23,15 @@ int main(int argc, char* argv[]) {
const string_view currentArg (argv[i]);
if (currentArg == ArgumentShort("printresult")) printResult = true;
else if (currentArg == ArgumentAssignable("target")) {
target.clear();
string value = ArgumentAssignable::getValueFor(currentArg.data());
const string value = ArgumentAssignable::getValueFor(currentArg.data());
if (!value.empty()) (target = '.') += value;
}
else if (currentArg == Argument("parallel")) parallel = true;
else if (currentArg.ends_with(".ybcon")) files.push_back(currentArg);
else if (currentArg == Argument("newlinesoff")) newLinesSeparator = true;
}
const auto compile = [&target](string_view name) {
string transpiledString = transpile(parseString(getFileContent(name.data())), target);
const auto compile = [&target, &newLinesSeparator](string_view name) {
string transpiledString = transpile(parseString(getFileContent(name.data()), newLinesSeparator ? ';' : '\n'), target);
name.remove_suffix(6);
string outputFile;
(outputFile = name).append(target);