308 lines
8.0 KiB
Go
308 lines
8.0 KiB
Go
// Package main provides a CLI utility to download HTML articles and convert them to Markdown with localized images.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"codeberg.org/readeck/go-readability/v2"
|
|
md "github.com/JohannesKaufmann/html-to-markdown"
|
|
"github.com/PuerkitoBio/goquery"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
const (
|
|
dirPerms = 0755
|
|
filePerms = 0644
|
|
)
|
|
|
|
var (
|
|
// slugRegex compiled once at package level for performance.
|
|
slugRegex = regexp.MustCompile("[^a-z0-9]+")
|
|
)
|
|
|
|
// ImageJob represents a single image to be downloaded concurrently.
|
|
type ImageJob struct {
|
|
URL string
|
|
LocalPath string
|
|
}
|
|
|
|
func main() {
|
|
var titleOverride string
|
|
var outDirBase string
|
|
|
|
flag.StringVar(&titleOverride, "title", "", "Override the article title and folder name")
|
|
flag.StringVar(&outDirBase, "out", ".", "Base directory to download the article folder into")
|
|
flag.Parse()
|
|
|
|
if flag.NArg() < 1 {
|
|
fmt.Fprintf(os.Stderr, "Usage: %s [options] <url>\nOptions:\n", os.Args[0])
|
|
flag.PrintDefaults()
|
|
os.Exit(1)
|
|
}
|
|
|
|
rawURL := flag.Arg(0)
|
|
|
|
// Handle graceful shutdown via signals.
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
client := &http.Client{Timeout: 30 * time.Second}
|
|
|
|
if err := run(ctx, client, rawURL, titleOverride, outDirBase); err != nil {
|
|
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
// run orchestrates the fetching, parsing, and downloading process.
|
|
func run(ctx context.Context, client *http.Client, rawURL, titleOverride, outDirBase string) error {
|
|
targetURL, err := url.Parse(rawURL)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse target URL %q: %w", rawURL, err)
|
|
}
|
|
|
|
fmt.Printf("Fetching: %s\n", rawURL)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to fetch URL: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("failed to fetch URL: status code %d", resp.StatusCode)
|
|
}
|
|
|
|
article, err := readability.FromReader(resp.Body, targetURL)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse article: %w", err)
|
|
}
|
|
|
|
finalTitle := article.Title()
|
|
if titleOverride != "" {
|
|
finalTitle = titleOverride
|
|
}
|
|
|
|
slug := generateSlug(finalTitle)
|
|
if slug == "" {
|
|
slug = "article"
|
|
}
|
|
|
|
outDir := filepath.Join(outDirBase, slug)
|
|
if err := os.MkdirAll(outDir, dirPerms); err != nil {
|
|
return fmt.Errorf("failed to create directory %s: %w", outDir, err)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := article.RenderHTML(&buf); err != nil {
|
|
return fmt.Errorf("failed to render article HTML: %w", err)
|
|
}
|
|
|
|
modifiedHTML, imageJobs, err := processHTML(buf.String(), targetURL, outDir)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to process HTML: %w", err)
|
|
}
|
|
|
|
converter := md.NewConverter("", true, nil)
|
|
markdownContent, err := converter.ConvertString(modifiedHTML)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to convert to markdown: %w", err)
|
|
}
|
|
|
|
frontmatter := fmt.Sprintf("---\ntitle: \"%s\"\nsource: \"%s\"\n---\n\n", finalTitle, rawURL)
|
|
markdownContent = frontmatter + markdownContent
|
|
|
|
if len(imageJobs) > 0 {
|
|
fmt.Printf("Downloading %d images concurrently...\n", len(imageJobs))
|
|
|
|
g, gCtx := errgroup.WithContext(ctx)
|
|
for _, job := range imageJobs {
|
|
g.Go(func() error {
|
|
return downloadImage(gCtx, client, job.URL, job.LocalPath)
|
|
})
|
|
}
|
|
|
|
if err := g.Wait(); err != nil {
|
|
fmt.Fprintf(os.Stderr, "warning: one or more images failed to download: %v\n", err)
|
|
}
|
|
}
|
|
|
|
mdPath := filepath.Join(outDir, "index.md")
|
|
if err := os.WriteFile(mdPath, []byte(markdownContent), filePerms); err != nil {
|
|
return fmt.Errorf("failed to write markdown file: %w", err)
|
|
}
|
|
|
|
fmt.Printf("Successfully saved article to: %s\n", mdPath)
|
|
return nil
|
|
}
|
|
|
|
// processHTML extracts images, rewrites their src attributes for a flat structure,
|
|
// and returns the modified HTML string and a slice of ImageJobs to be downloaded.
|
|
func processHTML(htmlContent string, targetURL *url.URL, outDir string) (string, []ImageJob, error) {
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
var imageJobs []ImageJob
|
|
|
|
doc.Find("img").Each(func(i int, s *goquery.Selection) {
|
|
src, exists := s.Attr("src")
|
|
if !exists || src == "" {
|
|
return
|
|
}
|
|
|
|
imgURL, err := targetURL.Parse(src)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "warning: failed to parse image URL %s: %v\n", src, err)
|
|
return
|
|
}
|
|
|
|
altText, _ := s.Attr("alt")
|
|
titleText, _ := s.Attr("title")
|
|
figCaption := s.ParentsFiltered("figure").Find("figcaption").Text()
|
|
|
|
// Traverse up DOM to find nearest preceding section header
|
|
var headerText string
|
|
curr := s
|
|
for curr.Length() > 0 && !curr.Is("body") {
|
|
h := curr.PrevAllFiltered("h1, h2, h3, h4, h5, h6").First()
|
|
if h.Length() > 0 {
|
|
headerText = h.Text()
|
|
break
|
|
}
|
|
curr = curr.Parent()
|
|
}
|
|
|
|
rawName := ""
|
|
if altText != "" {
|
|
rawName = altText
|
|
} else if titleText != "" {
|
|
rawName = titleText
|
|
} else if figCaption != "" {
|
|
rawName = figCaption
|
|
}
|
|
|
|
baseName := ""
|
|
if rawName != "" {
|
|
s.SetAttr("alt", strings.TrimSpace(rawName)) // set for markdown converter
|
|
baseName = generateSlug(rawName)
|
|
if len(baseName) > 40 {
|
|
baseName = baseName[:40]
|
|
baseName = strings.TrimRight(baseName, "-")
|
|
}
|
|
}
|
|
|
|
if baseName == "" || baseName == "-" {
|
|
baseNameStr := strings.TrimSuffix(filepath.Base(imgURL.Path), filepath.Ext(imgURL.Path))
|
|
baseName = generateSlug(baseNameStr)
|
|
|
|
// Fallback if filename is CDN hash (long) or empty
|
|
if len(baseName) >= 20 || baseName == "" || baseName == "-" || baseName == "image" {
|
|
if headerText != "" {
|
|
baseName = generateSlug(headerText)
|
|
s.SetAttr("alt", strings.TrimSpace(headerText))
|
|
} else {
|
|
baseName = fmt.Sprintf("section-image-%d", i+1)
|
|
s.SetAttr("alt", fmt.Sprintf("Section Image %d", i+1))
|
|
}
|
|
} else {
|
|
s.SetAttr("alt", strings.ReplaceAll(baseNameStr, "-", " "))
|
|
}
|
|
}
|
|
|
|
hash := sha256.Sum256([]byte(imgURL.String()))
|
|
hashStr := hex.EncodeToString(hash[:])[:6]
|
|
|
|
ext := filepath.Ext(imgURL.Path)
|
|
if ext == "" {
|
|
ext = ".jpg"
|
|
}
|
|
ext = strings.Split(ext, "?")[0]
|
|
|
|
// Format output like `capo-notation-a1b2c3.jpg` (using standard url-safe slugs)
|
|
filename := fmt.Sprintf("%s-%s%s", baseName, hashStr, ext)
|
|
localAbsPath := filepath.Join(outDir, filename)
|
|
|
|
s.SetAttr("src", filename)
|
|
|
|
imageJobs = append(imageJobs, ImageJob{
|
|
URL: imgURL.String(),
|
|
LocalPath: localAbsPath,
|
|
})
|
|
})
|
|
|
|
modifiedHTML, err := doc.Find("body").Html()
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("failed to render modified HTML: %w", err)
|
|
}
|
|
|
|
return modifiedHTML, imageJobs, nil
|
|
}
|
|
|
|
// downloadImage fetches the image from the given URL and saves it to localPath.
|
|
func downloadImage(ctx context.Context, client *http.Client, urlStr, localPath string) (err error) {
|
|
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
|
|
if reqErr != nil {
|
|
return fmt.Errorf("create request for %s: %w", urlStr, reqErr)
|
|
}
|
|
|
|
resp, doErr := client.Do(req)
|
|
if doErr != nil {
|
|
return fmt.Errorf("get %s: %w", urlStr, doErr)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("bad status %d for %s", resp.StatusCode, urlStr)
|
|
}
|
|
|
|
out, createErr := os.Create(localPath)
|
|
if createErr != nil {
|
|
return fmt.Errorf("create file %s: %w", localPath, createErr)
|
|
}
|
|
defer func() {
|
|
closeErr := out.Close()
|
|
if err == nil {
|
|
err = closeErr
|
|
}
|
|
}()
|
|
|
|
if _, err = io.Copy(out, resp.Body); err != nil {
|
|
return fmt.Errorf("write file %s: %w", localPath, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// generateSlug truncates the string to 5 words and converts it to a safe filesystem slug.
|
|
func generateSlug(s string) string {
|
|
words := strings.Fields(s)
|
|
if len(words) > 5 {
|
|
words = words[:5]
|
|
}
|
|
s = strings.Join(words, " ")
|
|
|
|
return strings.Trim(slugRegex.ReplaceAllString(strings.ToLower(s), "-"), "-")
|
|
}
|