Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
T
tx3-message-reader
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Packages
Packages
List
Container Registry
Analytics
Analytics
CI / CD
Code Review
Insights
Issues
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nanahira
tx3-message-reader
Commits
f1cbd75f
Commit
f1cbd75f
authored
Oct 03, 2020
by
nanahira
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
finish smbreader
parent
198d1278
Pipeline
#864
passed with stages
in 5 minutes and 42 seconds
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
65 additions
and
27 deletions
+65
-27
src/smbreader.ts
src/smbreader.ts
+65
-27
No files found.
src/smbreader.ts
View file @
f1cbd75f
...
...
@@ -2,8 +2,7 @@ import { Config, loadConfig } from "./config";
import
SMB
from
"
@marsaud/smb2
"
;
import
mysql
from
"
promise-mysql
"
;
import
moment
from
"
moment
"
;
import
iconv
from
"
iconv-lite
"
;
import
HTML
from
"
posthtml-parser
"
;
import
{
Message
,
readChatBuffer
}
from
"
./parse-buffer
"
;
export
class
SMBReader
{
...
...
@@ -42,33 +41,72 @@ export class SMBReader {
"
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
"
);
console
.
error
(
`
${
this
.
logPrefix
}
Initialization finished.`
);
}
private
getFilePath
(
filename
:
string
)
{
return
this
.
config
.
pathPrefix
?
`
${
this
.
config
.
pathPrefix
}
\\
${
filename
}
`
:
filename
;
}
private
flatternMessages
(
messages
:
Message
[]):
string
{
return
messages
.
map
(
m
=>
JSON
.
stringify
(
m
)).
join
(
"
|
"
);
}
async
run
()
{
console
.
error
(
`
${
this
.
logPrefix
}
Reading file list...`
);
let
filelist
=
(
await
this
.
smb
.
readdir
(
this
.
config
.
pathPrefix
)).
filter
(
m
=>
m
.
match
(
/^chat_.*
\.
html$/
));
const
files
=
filelist
.
map
(
filename
=>
{
const
dateString
=
filename
.
match
(
/^chat_
(
.*
)\.
html$/
)[
1
].
replace
(
/-/g
,
"
:
"
);
const
date
=
moment
(
dateString
).
utcOffset
(
"
+08:00
"
,
true
)
return
{
name
:
filename
,
date
}
});
files
.
sort
((
f1
,
f2
)
=>
{
return
f1
.
date
.
unix
()
-
f2
.
date
.
unix
();
});
for
(
let
file
of
files
)
{
const
testIndex
:
any
[]
=
await
this
.
db
.
query
(
"
select * from `filesRead` where filename = ?
"
,
[
file
.
name
]);
if
(
testIndex
.
length
)
{
console
.
error
(
`
${
this
.
logPrefix
}
File
${
file
.
name
}
has already been read. Skipping.`
);
continue
;
}
console
.
error
(
`
${
this
.
logPrefix
}
Reading file
${
file
.
name
}
...`
);
const
fileMetadata
=
await
this
.
db
.
query
(
"
insert into `filesRead` set ?
"
,
{
filename
:
file
.
name
,
date
:
file
.
date
.
format
(
"
YYYY-MM-DD HH:mm:ss
"
)
try
{
console
.
error
(
`
${
this
.
logPrefix
}
Reading file list...`
);
let
filelist
=
(
await
this
.
smb
.
readdir
(
this
.
config
.
pathPrefix
)).
filter
(
m
=>
m
.
match
(
/^chat_.*
\.
html$/
));
const
files
=
filelist
.
map
(
filename
=>
{
const
dateString
=
filename
.
match
(
/^chat_
(
.*
)\.
html$/
)[
1
].
replace
(
/-/g
,
"
:
"
);
const
date
=
moment
(
dateString
).
utcOffset
(
"
+08:00
"
,
true
)
return
{
name
:
filename
,
date
}
});
const
insertedFileID
:
number
=
fileMetadata
.
insertId
;
files
.
sort
((
f1
,
f2
)
=>
{
return
f1
.
date
.
unix
()
-
f2
.
date
.
unix
();
});
for
(
let
file
of
files
)
{
const
filePath
=
this
.
getFilePath
(
file
.
name
);
try
{
const
testIndex
:
any
[]
=
await
this
.
db
.
query
(
"
select * from `filesRead` where filename = ?
"
,
[
file
.
name
]);
if
(
testIndex
.
length
)
{
console
.
error
(
`
${
this
.
logPrefix
}
File
${
filePath
}
has already been read. Skipping.`
);
continue
;
}
console
.
error
(
`
${
this
.
logPrefix
}
Reading file
${
filePath
}
...`
);
const
fileMetadata
=
await
this
.
db
.
query
(
"
insert into `filesRead` set ?
"
,
{
filename
:
file
.
name
,
date
:
file
.
date
.
format
(
"
YYYY-MM-DD HH:mm:ss
"
)
});
const
insertedFileID
:
number
=
fileMetadata
.
insertId
;
const
previousFilenameArray
:
any
[]
=
await
this
.
db
.
query
(
"
select filename from `filesRead` where fileid = ?
"
,
[
insertedFileID
-
1
]);
let
currentMessages
=
readChatBuffer
((
await
this
.
smb
.
readFile
(
filePath
))
as
Buffer
);
if
(
previousFilenameArray
.
length
&&
files
.
find
(
f
=>
f
.
name
===
previousFilenameArray
[
0
].
filename
))
{
const
previousFilePath
=
this
.
getFilePath
(
previousFilenameArray
[
0
].
filename
);
console
.
error
(
`
${
this
.
logPrefix
}
Comparing file
${
filePath
}
with old file
${
previousFilePath
}
.`
)
try
{
const
previousMessages
=
readChatBuffer
((
await
this
.
smb
.
readFile
(
previousFilePath
))
as
Buffer
);
const
flattenedPreviousMessages
=
this
.
flatternMessages
(
previousMessages
);
const
flattenedCurrentMessages
=
this
.
flatternMessages
(
currentMessages
);
if
(
flattenedCurrentMessages
.
startsWith
(
flattenedPreviousMessages
))
{
const
sliceCount
=
previousMessages
.
length
;
console
.
error
(
`
${
this
.
logPrefix
}
Slicing the first
${
sliceCount
}
messages of file
${
filePath
}
.`
);
currentMessages
=
currentMessages
.
slice
(
sliceCount
);
}
}
catch
(
e
)
{
console
.
error
(
`
${
this
.
logPrefix
}
Failed comparing file
${
filePath
}
with old file
${
previousFilePath
}
. Skipping:`
,
e
.
toString
());
}
}
for
(
let
message
of
currentMessages
)
{
console
.
error
(
`
${
this
.
logPrefix
}
Recording message from file
${
insertedFileID
}
${
filePath
}
:`
,
JSON
.
stringify
(
message
));
await
this
.
db
.
query
(
"
insert into `messages` set ?
"
,
{
fileid
:
insertedFileID
,
...
message
});
}
}
catch
(
e
)
{
console
.
error
(
`
${
this
.
logPrefix
}
Errored reading file
${
filePath
}
:`
,
e
.
toString
());
}
}
}
catch
(
e
)
{
console
.
error
(
`
${
this
.
logPrefix
}
Read error:`
,
e
.
toString
());
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment