twocookedfaggots/scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: twocookedfaggots:3bbd6c72ca4650fa65a44114da1b26b1c4019f7d
Branches Tags
twocookedfaggots:master
..
compare: twocookedfaggots:78985687a17dd5d970274938a2a4809e9c1a0112
Branches Tags
twocookedfaggots:master

2 Commits
3bbd6c72ca ... 78985687a1

Author SHA1 Message Date
potassium
78985687a1 go.mod 2026-01-04 13:25:59 +03:00
potassium
5cb8665a08 exported 2025-12-27 09:07:39 +03:00
3 changed files with 6 additions and 13 deletions
Expand all files Collapse all files

5
go.mod
View File

@@ -1,5 +0,0 @@
module git.nkpl.cc/XoxJlopeZi4BB/scraper.git
go 1.25.5
require golang.org/x/net v0.48.0

2
go.sum
View File

@@ -1,2 +0,0 @@
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=

12
scraper.go
View File

@@ -8,7 +8,7 @@ import (
type crawlFunc func(*html.Node) type crawlFunc func(*html.Node)
func searchElem(n *html.Node, data string) chan *html.Node { func SearchElem(n *html.Node, data string) chan *html.Node {
ch := make(chan *html.Node) ch := make(chan *html.Node)
var crawl crawlFunc var crawl crawlFunc
crawl = func(n *html.Node) { crawl = func(n *html.Node) {
@@ -29,7 +29,7 @@ func searchElem(n *html.Node, data string) chan *html.Node {
return ch return ch
} }
func searchAttr(n *html.Node, key, contains string) chan *html.Node { func SearchAttr(n *html.Node, key, contains string) chan *html.Node {
ch := make(chan *html.Node) ch := make(chan *html.Node)
var crawl crawlFunc var crawl crawlFunc
crawl = func(n *html.Node) { crawl = func(n *html.Node) {
@@ -52,14 +52,14 @@ func searchAttr(n *html.Node, key, contains string) chan *html.Node {
return ch return ch
} }
func searchElemAttr(n *html.Node, elem, key, value string) chan *html.Node { func SearchElemAttr(n *html.Node, elem, key, value string) chan *html.Node {
ch := make(chan *html.Node) ch := make(chan *html.Node)
go func() { go func() {
defer close(ch) defer close(ch)
for e := range searchElem(n, elem) { for e := range SearchElem(n, elem) {
// If document is too large there are // If document is too large there are
// would be a hundreds of goroutines :(( // would be a hundreds of goroutines :((
for attr := range searchAttr(e, key, value) { for attr := range SearchAttr(e, key, value) {
ch <- attr ch <- attr
} }
} }
@@ -67,7 +67,7 @@ func searchElemAttr(n *html.Node, elem, key, value string) chan *html.Node {
return ch return ch
} }
func crawlText(n *html.Node) string { func CrawlText(n *html.Node) string {
var s = new(strings.Builder) var s = new(strings.Builder)
var crawl crawlFunc var crawl crawlFunc
crawl = func(n *html.Node) { crawl = func(n *html.Node) {