You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

299 lines
7.0 KiB

package favicon
// draws heavily on Scott Werner's python package https://github.com/scottwernervt/favicon
// todo: separate into its own module
import (
"errors"
"fmt"
"github.com/PuerkitoBio/goquery"
"io"
"io/ioutil"
"log"
"mime"
"net/http"
"net/url"
"path"
"regexp"
"sort"
"strconv"
"strings"
"time"
)
// Icon describes a favicon or other icon for a given website
type Icon struct {
width int
height int
RemoteUrl string
FilePath string
mimeType string
size int64
PageTitle string
}
var linkRels = [4]string{"icon", "shortcut icon", "apple-touch-icon", "apple-touch-icon-precomposed"}
var metaNames = [3]string{"msapplication-TileImage", "og:image", "image"}
// GetBest takes a url and gets the best Icon for it (where best is defined as largest file size)
func GetBest(getUrl string) (*Icon, error) {
parsedUrl, err := url.Parse(getUrl)
if err != nil {
return nil, err
}
req := http.Request{
Method: "GET",
URL: parsedUrl,
Proto: "HTTP",
}
c := http.Client{
Timeout: 4 * time.Second,
}
resp, err := c.Do(&req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return nil, errors.New("received bad status code")
}
var icons []*Icon
respUrl := resp.Request.URL
defIco, err := defaultIcon(respUrl)
if err == nil {
icons = append(icons, defIco)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
log.Print(err)
return nil, err
}
tmIcons, err := tagMetaIcons(*doc, *respUrl)
if err == nil {
icons = append(icons, tmIcons...)
}
if len(icons) == 0 {
return nil, errors.New("failed to get any icons for website")
}
log.Printf("favicon finder got a total of %d icons to choose from", len(icons))
// get the largest favicon
sort.Slice(icons, func(i, j int) bool {
return icons[i].size > icons[j].size
})
bestIcon := icons[0]
bestIcon.PageTitle = getTitle(*doc, *respUrl)
return bestIcon, nil
}
func getTitle(doc goquery.Document, respUrl url.URL) string {
title := strings.TrimSpace(doc.Find("title").First().Text())
log.Printf("Got title '%s' for page at %s", title, respUrl.String())
return title
}
func getTagsToFetch(doc goquery.Document) []*goquery.Selection {
var tagsToFetch []*goquery.Selection
// handle link nodes
doc.Find("link").Each(func(i int, s *goquery.Selection) {
rel, exists := s.Attr("rel")
if !exists {
return
}
rel = strings.ToLower(rel)
for _, lr := range linkRels {
if rel == lr {
tagsToFetch = append(tagsToFetch, s)
return
}
}
})
// handle meta nodes
doc.Find("meta").Each(func(i int, s *goquery.Selection) {
metaType, exists := s.Attr("name")
if !exists {
metaType, exists = s.Attr("property")
if !exists {
metaType, exists = s.Attr("itemprop")
if !exists {
return
}
}
}
metaType = strings.ToLower(metaType)
for _, mn := range metaNames {
if metaType == strings.ToLower(mn) {
tagsToFetch = append(tagsToFetch, s)
}
}
})
return tagsToFetch
}
func tagMetaIcons(doc goquery.Document, respUrl url.URL) ([]*Icon, error) {
tagsToFetch := getTagsToFetch(doc)
log.Printf("Got %d icon tags for RemoteUrl %s", len(tagsToFetch), respUrl.String())
var icons []*Icon
for _, s := range tagsToFetch {
linkUrl, err := tagMetaUrlGetter(s)
if err != nil {
continue
}
trimmedLinkUrl := strings.TrimSpace(*linkUrl)
if trimmedLinkUrl == "" || strings.HasPrefix(trimmedLinkUrl, "data:image/") {
continue
}
parsedLinkUrl, err := url.Parse(trimmedLinkUrl)
if err != nil {
continue
}
if !parsedLinkUrl.IsAbs() {
parsedLinkUrl.Host = respUrl.Host
parsedLinkUrl.Scheme = respUrl.Scheme
if strings.HasPrefix(parsedLinkUrl.Path, ".") {
parsedLinkUrl.Path = path.Join(respUrl.Path, parsedLinkUrl.Path)
}
}
width, height := dimensions(s, linkUrl)
downloadedIcon, err := getIconFromUrl(parsedLinkUrl)
if err != nil {
continue
}
icons = append(icons, &Icon{
width: width,
height: height,
RemoteUrl: downloadedIcon.RemoteUrl,
FilePath: downloadedIcon.FilePath,
mimeType: downloadedIcon.mimeType,
size: downloadedIcon.size,
})
}
return icons, nil
}
func dimensions(t *goquery.Selection, linkUrl *string) (int, int) {
// Get icon dimensions from size attribute or filename
sizes, exists := t.Attr("sizes")
var width, height string
if exists && sizes != "any" {
size := strings.Split(sizes, " ")
sort.Sort(sort.Reverse(sort.StringSlice(size)))
re := regexp.MustCompile(`(?m)[x\xd7]`)
unpack(re.Split(size[0], 2), &width, &height)
} else {
sizeRE := regexp.MustCompile("(?mi)(?P<width>[[:digit:]]{2,4})x(?P<height>[[:digit:]]{2,4})")
matches := sizeRE.FindStringSubmatch(*linkUrl)
if matches != nil && len(matches) == 2 {
unpack(matches, &width, &height)
} else {
width, height = "0", "0"
}
}
// repair bad attribute values e.g. 192x192+
numRE := regexp.MustCompile("[0-9]+")
width = numRE.FindString(width)
height = numRE.FindString(height)
widthInt, err := strconv.Atoi(width)
if err != nil {
widthInt = 0
}
heightInt, err := strconv.Atoi(height)
if err != nil {
heightInt = 0
}
// todo: support og:image:width content=1000 - requires doc level lookups
return widthInt, heightInt
}
func tagMetaUrlGetter(tm *goquery.Selection) (*string, error) {
href, exists := tm.Attr("href")
if exists {
return &href, nil
}
content, exists := tm.Attr("content")
if exists {
return &content, nil
}
return nil, errors.New("no RemoteUrl found on html tag")
}
func responseToFile(response http.Response, extension string) (string, *int64) {
file, err := ioutil.TempFile("", fmt.Sprintf("portfall*%s", extension))
if err != nil {
log.Fatal(err)
}
defer file.Close()
size, err := io.Copy(file, response.Body)
if err != nil {
log.Fatal(err)
}
return file.Name(), &size
}
func defaultIcon(parsedUrl *url.URL) (*Icon, error) {
faviconUrl := url.URL{
Scheme: parsedUrl.Scheme,
Host: parsedUrl.Host,
Path: "favicon.ico",
}
icon, err := getIconFromUrl(&faviconUrl)
return icon, err
}
func getIconFromUrl(iconUrl *url.URL) (*Icon, error) {
// download icon and get extension and size
log.Printf("Getting icon from RemoteUrl %s", iconUrl.String())
c := http.Client{
Timeout: 3 * time.Second,
}
resp, err := c.Get(iconUrl.String())
if err != nil {
return nil, err
}
if resp.StatusCode >= 400 {
return nil, fmt.Errorf(
"received bad status code %d attempting to retrieve icon at %s", resp.StatusCode, iconUrl.String())
}
defer resp.Body.Close()
mimeType := resp.Header.Get("content-type")
extensions, err := mime.ExtensionsByType(mimeType)
if err != nil {
return nil, err
}
var extension string
if extensions != nil {
extension = extensions[0]
}
if !strings.HasPrefix(mimeType, "image") {
return nil, fmt.Errorf("bad mimeType %s on remote icon", mimeType)
}
fp, size := responseToFile(*resp, extension)
return &Icon{
width: 0,
height: 0,
RemoteUrl: iconUrl.String(),
FilePath: fp,
mimeType: mimeType,
size: *size,
}, nil
}
func unpack(s []string, vars ...*string) {
for i, str := range s {
*vars[i] = str
}
}