ccppi/nim-basic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

comparis has a strange dynamicaly loading js bullschit thing therefore the scrapper works but does not get all the data which is seen in the browser

Browse Source
This commit is contained in:
ccppi
2024-07-02 14:00:19 +02:00
parent f629ff784f
commit a2a0001aef
3 changed files with 27 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
scrap.nim
View File

@@ -12,6 +12,7 @@ type
html_context_key* : string
contains_string* : string
aattrs* : string
attrs_key*: string
content* : seq[string]
type
Entry* = object
@@ -40,32 +41,42 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
if c.attrs.hasKey(entry.desc[i].html_context_attrs):
echo("c.attrs[key]: ",c.attrs[entry.desc[i].html_context_attrs])
if entry.desc[i].html_context_key != "":
if entry.desc[i].html_context_key.contains(c.attrs[entry.desc[i].html_context_attrs]):
#if entry.desc[i].html_context_key.contains(c.attrs[entry.desc[i].html_context_attrs]):
if c.attrs[entry.desc[i].html_context_attrs].contains(entry.desc[i].html_context_key):
context.add(c)
echo("context_key_ MATCH")
else:
echo("key does NOT match")
else:
context.add(c)
else:
echo("c.attrs does not have the key")
echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs)
else:
echo("no attributes on this tag")
echo("out of for loop")
echo("----------CONTEXT RESULT")
echo(context)
#end filter context
for a in context:
let subContext = a.findAll(entry.desc[i].html_tag)
for b in subContext:
if entry.desc[i].aattrs != "":
if b.attrs.hasKey(entry.desc[i].aattrs):
echo("found key [",entry.desc[i].aattrs,"] in : ",b)
echo("value: ",b.attrs[entry.desc[i].aattrs])
if entry.desc[i].contains_string != "":
var content_txt :string = b.innerText()
echo(" and string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("PATTERN MATCH")
entry.desc[i].content.add(content_txt)
else:
entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
if b.attrs != nil:
if b.attrs.hasKey(entry.desc[i].aattrs):
echo("found key [",entry.desc[i].aattrs,"] in : ",b)
echo("value: ",b.attrs[entry.desc[i].aattrs])
if entry.desc[i].contains_string != "":
var content_txt :string = b.innerText()
echo(" and string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("ATTRS MATCH")
entry.desc[i].content.add(content_txt)
elif entry.desc[i].attrs_key != "":
#if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
echo("KEY_MATCH")
entry.desc[i].content.add(b.innerText())
else:
entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
elif entry.desc[i].contains_string != "":
var content_txt = b.innerText()
echo(" string(innerText): ",content_txt)