10
20
30
40
50
60
70
80
90
100
110
120
130
140
150
160
170
180
190
200
210
220
230
240
250
260
272
280
291
300
310
321
330
340
350
360
372
380
3921
400
411
420
430
440
450
461
470
480
490
500
510
520
530
540
550
560
570
580
590
600
610
620
630
640
650
660
670
680
690
700
710
720
730
740
750
760
770
780
790
800
810
82169
830
841411
850
86400
870
88148
890
900
91252
920
930
94169
950
960
970
980
9944
10044
1010
102765
1030
104211
1050
106109
1070
1080
109211
1100
11144
1120
1130
1140
11544
1160
1170
1180
1190
1204
1210
1220
1230
1240
1250
1260
1270
1280
1290
1300
1310
1322
1330
1340
1352
1360
1372
1380
1390
1400
1410
14232
1430
1440
1450
1460
1478
1480
1490
1500
1510
15226
1530
1546
1550
1562
1570
1580
1590
1602
1610
1620
1630
1640
1650
1660
1670
1680
1690
1700
1710
1720
1730
1740
1750
1766
1776
1780
1790
1800
1816
1820
1832
1840
1850
1866
1870
1880
1890
1900
1916
1920
1936
1946
1950
1966
1976
1986
1996
2006
2010
2026
2030
2042510
2052510
2060
2072510
2080
20914
2100
2110
2122510
2130
21474
2150
2160
2172510
2180
21974
2200
22174
2220
22320
2240
2250
2260
2272510
2280
2295214
2300
23180
23280
2330
2340
2350
2366
2370
2380
2390
2400
2410
2420
2430
2440
2450
2460
2470
2480
2490
2500
2510
25255
2530
25455834
2550
25655779
25755779
25855779
2590
2600
2610
2620
2630
26455
2650
2660
2670
2680
2690
27090
2710
27290
273174
2740
27572
2760
2770
27818
2790
2800
28172
2820
2830
2840
2850
2860
287168
2880
2890
2900
2910
2920
2930
2940
2950
2960
2970
2980
2990
3000
3010
3020
3030
3040
3050
3060
3070
30814
30914
3100
3110
3120
3130
31452
3150
31652
31752
3180
3195152
3200
3215152
3220
3235152
3240
325284
326284
3270
3280
3295152
3300
331284
3320
3330
3345152
3355152
33610160
3370
33852
3390
3400
3410
34252
3430
3440
3450
3460
3470
348364
3490
350364
3510
352117
3530
3540
355247
3560
3570
358117
3590
3600
3610
3620
3630
36489
3650
36689
3670
3680
3690
3700
3710
3720
37319
3740
375123
3760
377123
3780
37919
3800
3810
382208
383104
3840
3850
38619
3870
3880
3890
3900
3910
3920
3936
3940
3956
3966
3970
3982578
3990
4002578
4010
4022578
4030
40474
40574
4060
4070
4082578
4090
41074
4110
4122578
4132578
4145130
4150
4166
4170
4180
4190
4206
4210
4220
4230
4240
4250
4260
42763
4280
42963
43063
43163
4320
433280
4340
435280
4360
437301
4380
4397
4400
4410
442287
4430
4440
4450
4460
447287
4480
4497
4500
4510
452273
4530
45456
45556
45656
4570
4580
459273
4600
46156
4620
4630
464273
4650
466450
4670
46856
4690
4700
471217
4720
4730
47463
4750
4760
4770
4780
4790
4800
4810
4827
4830
484252
4850
48677
4870
48814
4890
4900
4910
49263
4930
4940
4950
4967
4970
/++
A module containing parsing code utilities
Copyright: © 2017 Szabo Bogdan
License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
Authors: Szabo Bogdan
+/
module trial.discovery.code;
import std.algorithm;
import std.string;
import std.range;
import std.file;
import std.stdio;
import std.conv;
version (Have_libdparse)
{
public import dparse.ast;
public import dparse.lexer;
public import dparse.parser;
}
/// Get the module name of a DLang source file
string getModuleName(string fileName)
{
if (!exists(fileName))
{
return "";
}
if (isDir(fileName))
{
return "";
}
auto file = File(fileName);
auto moduleLine = file.byLine().map!(a => a.to!string).filter!(a => a.startsWith("module"));
if (moduleLine.empty)
{
return "";
}
return moduleLine.front.split(' ')[1].split(";")[0];
}
version (Have_libdparse)
{
const(Token)[] stringToDTokens(string data)
{
try {
ubyte[] fileBytes = cast(ubyte[]) data;
StringCache cache = StringCache(StringCache.defaultBucketCount);
LexerConfig config;
config.stringBehavior = StringBehavior.source;
config.fileName = "";
config.commentBehavior = CommentBehavior.intern;
auto lexer = DLexer(fileBytes, config, &cache);
const(Token)[] tokens = lexer.array;
return tokens.map!(token => const Token(token.type, token.text.idup, token.line, token.column, token.index)).array;
} catch(Throwable t) {
t.writeln;
return [];
}
}
///
struct DLangAttribute
{
const(Token)[] tokens;
inout
{
string identifier()
{
string result;
foreach (token; tokens)
{
if (str(token.type) == "(")
{
break;
}
result ~= token.text;
}
return result;
}
string value()
{
bool after;
string result;
foreach (token; tokens)
{
if (after)
{
result ~= token.text.strip('"').strip('`').strip('\'');
}
if (str(token.type) == "(")
{
after = true;
}
}
return result;
}
auto line()
{
return tokens[0].line;
}
}
}
struct DLangFunction
{
const(DLangAttribute)[] attributes;
const(Token)[] tokens;
string name()
{
auto result = TokenIterator(tokens).readUntilType("(").replace("\n", " ")
.replace("\r", " ").replace("\t", " ").split(" ");
std.algorithm.reverse(result);
return result[0];
}
bool hasAttribute(string name)
{
return !attributes.filter!(a => a.identifier == name).empty;
}
auto getAttribute(string name)
{
return attributes.filter!(a => a.identifier == name).front;
}
string testName()
{
foreach (attribute; attributes)
{
if (attribute.identifier == "")
{
return attribute.value;
}
}
return name.camelToSentence;
}
size_t line()
{
return TokenIterator(tokens).skipUntilType("(").currentToken.line;
}
}
struct DLangClass
{
const(Token)[] tokens;
/// returns the class name
string name()
{
auto iterator = TokenIterator(tokens);
auto name = iterator.readUntilType("{");
import std.stdio;
if (name.indexOf(":") != -1)
{
name = name.split(":")[0];
}
return name.strip;
}
DLangFunction[] functions()
{
int paranthesisCount;
auto iterator = TokenIterator(tokens);
iterator.skipUntilType("{");
const(Token)[] currentTokens;
DLangFunction[] discoveredFunctions;
DLangAttribute[] attributes;
bool readingFunction;
int functionLevel = 1;
foreach (token; iterator)
{
string type = token.type.str;
currentTokens ~= token;
if (type == "@")
{
attributes ~= iterator.readAttribute;
}
if (type == "{")
{
paranthesisCount++;
}
if (type == "}")
{
paranthesisCount--;
if (paranthesisCount == functionLevel)
{
discoveredFunctions ~= DLangFunction(attributes, currentTokens);
}
}
readingFunction = paranthesisCount > functionLevel;
if (type == "}" || (!readingFunction && type == ";"))
{
currentTokens = [];
attributes = [];
}
}
return discoveredFunctions;
}
}
/// An iterator that helps to deal with DLang tokens
struct TokenIterator
{
private
{
const(Token)[] tokens;
size_t index;
}
///
int opApply(int delegate(const(Token)) dg)
{
int result = 0;
while (index < tokens.length)
{
result = dg(tokens[index]);
index++;
if (result)
{
break;
}
}
return result;
}
///
ref auto skipWsAndComments()
{
while (index < tokens.length)
{
auto type = str(tokens[index].type);
if (type != "comment" && type != "whitespace")
{
break;
}
index++;
}
return this;
}
///
auto currentToken()
{
return tokens[index];
}
/// Skip until a token with a certain text is reached
ref auto skipUntil(string text)
{
while (index < tokens.length)
{
if (tokens[index].text == text)
{
break;
}
index++;
}
return this;
}
ref auto skipNextBlock()
{
readNextBlock();
return this;
}
auto readNextBlock()
{
const(Token)[] blockTokens = [];
bool readingBlock;
int paranthesisCount;
while (index < tokens.length)
{
auto type = str(tokens[index].type);
if (type == "{")
{
paranthesisCount++;
readingBlock = true;
}
if (type == "}")
{
paranthesisCount--;
}
blockTokens ~= tokens[index];
index++;
if (readingBlock && paranthesisCount == 0)
{
break;
}
}
return blockTokens;
}
/// Skip until a token with a certain type is reached
ref auto skipUntilType(string type)
{
while (index < tokens.length)
{
if (str(tokens[index].type) == type)
{
break;
}
index++;
}
return this;
}
/// Skip one token
ref auto skipOne()
{
index++;
return this;
}
/// Concatenate all the tokens until the first token of a certain type
/// that will be ignored
string readUntilType(string type)
{
string result;
while (index < tokens.length)
{
if (str(tokens[index].type) == type)
{
break;
}
result ~= tokens[index].text == "" ? str(tokens[index].type) : tokens[index].text;
index++;
}
return result;
}
/// Returns a Dlang class. You must call this method after the
/// class token was read.
DLangClass readClass()
{
const(Token)[] classTokens = [];
bool readingClass;
int paranthesisCount;
while (index < tokens.length)
{
auto type = str(tokens[index].type);
if (type == "{")
{
paranthesisCount++;
readingClass = true;
}
if (type == "}")
{
paranthesisCount--;
}
classTokens ~= tokens[index];
index++;
if (readingClass && paranthesisCount == 0)
{
break;
}
}
return DLangClass(classTokens);
}
/// Returns a Dlang attribute. You must call this method after the
/// @ token was read.
DLangAttribute readAttribute()
{
const(Token)[] attributeTokens = [];
int paranthesisCount;
bool readingParams;
bool foundWs;
while (index < tokens.length)
{
auto type = str(tokens[index].type);
if (type == "whitespace" && paranthesisCount == 0 && !readingParams)
{
foundWs = true;
}
if (foundWs && type == ".")
{
foundWs = false;
}
if (foundWs && type != "(")
{
break;
}
if (type == "(")
{
paranthesisCount++;
readingParams = true;
foundWs = false;
}
if (type == ")")
{
paranthesisCount--;
}
attributeTokens ~= tokens[index];
if (readingParams && paranthesisCount == 0)
{
break;
}
index++;
}
return DLangAttribute(attributeTokens);
}
}
}
/// Converts a string from camel notation to a readable sentence
string camelToSentence(const string name) pure
{
string sentence;
foreach (ch; name)
{
if (ch.toUpper == ch)
{
sentence ~= " " ~ ch.toLower.to!string;
}
else
{
sentence ~= ch;
}
}
return sentence.capitalize;
}