v / examples / web_crawler / web_crawler.v
21 lines · 20 sloc · 567 bytes · c51d30bf5309653c6b573ec815268e69a78ea8cc
Raw
1import net.http
2import net.html
3
4fn main() {
5 site_url := 'https://news.ycombinator.com'
6 resp := http.fetch(
7 url: site_url
8 user_agent: 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
9 )!
10 mut doc := html.parse(resp.body)
11 tags := doc.get_tags_by_attribute_value('class', 'titleline')
12 for i, tag in tags {
13 el := tag.children[0]
14 mut href := el.attributes['href']!
15 if !href.starts_with('http') {
16 href = '${site_url}/${href}'
17 }
18 title := el.content
19 println('${i + 1:2}. title: ${title:-90s} href: ${href}')
20 }
21}
22