adding tool and tests
This commit is contained in:
307
main.go
Normal file
307
main.go
Normal file
@@ -0,0 +1,307 @@
|
||||
// Package main provides a CLI utility to download HTML articles and convert them to Markdown with localized images.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"codeberg.org/readeck/go-readability/v2"
|
||||
md "github.com/JohannesKaufmann/html-to-markdown"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
dirPerms = 0755
|
||||
filePerms = 0644
|
||||
)
|
||||
|
||||
var (
|
||||
// slugRegex compiled once at package level for performance.
|
||||
slugRegex = regexp.MustCompile("[^a-z0-9]+")
|
||||
)
|
||||
|
||||
// ImageJob represents a single image to be downloaded concurrently.
|
||||
type ImageJob struct {
|
||||
URL string
|
||||
LocalPath string
|
||||
}
|
||||
|
||||
func main() {
|
||||
var titleOverride string
|
||||
var outDirBase string
|
||||
|
||||
flag.StringVar(&titleOverride, "title", "", "Override the article title and folder name")
|
||||
flag.StringVar(&outDirBase, "out", ".", "Base directory to download the article folder into")
|
||||
flag.Parse()
|
||||
|
||||
if flag.NArg() < 1 {
|
||||
fmt.Fprintf(os.Stderr, "Usage: %s [options] <url>\nOptions:\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
rawURL := flag.Arg(0)
|
||||
|
||||
// Handle graceful shutdown via signals.
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
if err := run(ctx, client, rawURL, titleOverride, outDirBase); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run orchestrates the fetching, parsing, and downloading process.
|
||||
func run(ctx context.Context, client *http.Client, rawURL, titleOverride, outDirBase string) error {
|
||||
targetURL, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse target URL %q: %w", rawURL, err)
|
||||
}
|
||||
|
||||
fmt.Printf("Fetching: %s\n", rawURL)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch URL: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("failed to fetch URL: status code %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
article, err := readability.FromReader(resp.Body, targetURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse article: %w", err)
|
||||
}
|
||||
|
||||
finalTitle := article.Title()
|
||||
if titleOverride != "" {
|
||||
finalTitle = titleOverride
|
||||
}
|
||||
|
||||
slug := generateSlug(finalTitle)
|
||||
if slug == "" {
|
||||
slug = "article"
|
||||
}
|
||||
|
||||
outDir := filepath.Join(outDirBase, slug)
|
||||
if err := os.MkdirAll(outDir, dirPerms); err != nil {
|
||||
return fmt.Errorf("failed to create directory %s: %w", outDir, err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := article.RenderHTML(&buf); err != nil {
|
||||
return fmt.Errorf("failed to render article HTML: %w", err)
|
||||
}
|
||||
|
||||
modifiedHTML, imageJobs, err := processHTML(buf.String(), targetURL, outDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to process HTML: %w", err)
|
||||
}
|
||||
|
||||
converter := md.NewConverter("", true, nil)
|
||||
markdownContent, err := converter.ConvertString(modifiedHTML)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to convert to markdown: %w", err)
|
||||
}
|
||||
|
||||
frontmatter := fmt.Sprintf("---\ntitle: \"%s\"\nsource: \"%s\"\n---\n\n", finalTitle, rawURL)
|
||||
markdownContent = frontmatter + markdownContent
|
||||
|
||||
if len(imageJobs) > 0 {
|
||||
fmt.Printf("Downloading %d images concurrently...\n", len(imageJobs))
|
||||
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
for _, job := range imageJobs {
|
||||
g.Go(func() error {
|
||||
return downloadImage(gCtx, client, job.URL, job.LocalPath)
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: one or more images failed to download: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
mdPath := filepath.Join(outDir, "index.md")
|
||||
if err := os.WriteFile(mdPath, []byte(markdownContent), filePerms); err != nil {
|
||||
return fmt.Errorf("failed to write markdown file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Successfully saved article to: %s\n", mdPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// processHTML extracts images, rewrites their src attributes for a flat structure,
|
||||
// and returns the modified HTML string and a slice of ImageJobs to be downloaded.
|
||||
func processHTML(htmlContent string, targetURL *url.URL, outDir string) (string, []ImageJob, error) {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
var imageJobs []ImageJob
|
||||
|
||||
doc.Find("img").Each(func(i int, s *goquery.Selection) {
|
||||
src, exists := s.Attr("src")
|
||||
if !exists || src == "" {
|
||||
return
|
||||
}
|
||||
|
||||
imgURL, err := targetURL.Parse(src)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: failed to parse image URL %s: %v\n", src, err)
|
||||
return
|
||||
}
|
||||
|
||||
altText, _ := s.Attr("alt")
|
||||
titleText, _ := s.Attr("title")
|
||||
figCaption := s.ParentsFiltered("figure").Find("figcaption").Text()
|
||||
|
||||
// Traverse up DOM to find nearest preceding section header
|
||||
var headerText string
|
||||
curr := s
|
||||
for curr.Length() > 0 && !curr.Is("body") {
|
||||
h := curr.PrevAllFiltered("h1, h2, h3, h4, h5, h6").First()
|
||||
if h.Length() > 0 {
|
||||
headerText = h.Text()
|
||||
break
|
||||
}
|
||||
curr = curr.Parent()
|
||||
}
|
||||
|
||||
rawName := ""
|
||||
if altText != "" {
|
||||
rawName = altText
|
||||
} else if titleText != "" {
|
||||
rawName = titleText
|
||||
} else if figCaption != "" {
|
||||
rawName = figCaption
|
||||
}
|
||||
|
||||
baseName := ""
|
||||
if rawName != "" {
|
||||
s.SetAttr("alt", strings.TrimSpace(rawName)) // set for markdown converter
|
||||
baseName = generateSlug(rawName)
|
||||
if len(baseName) > 40 {
|
||||
baseName = baseName[:40]
|
||||
baseName = strings.TrimRight(baseName, "-")
|
||||
}
|
||||
}
|
||||
|
||||
if baseName == "" || baseName == "-" {
|
||||
baseNameStr := strings.TrimSuffix(filepath.Base(imgURL.Path), filepath.Ext(imgURL.Path))
|
||||
baseName = generateSlug(baseNameStr)
|
||||
|
||||
// Fallback if filename is CDN hash (long) or empty
|
||||
if len(baseName) >= 20 || baseName == "" || baseName == "-" || baseName == "image" {
|
||||
if headerText != "" {
|
||||
baseName = generateSlug(headerText)
|
||||
s.SetAttr("alt", strings.TrimSpace(headerText))
|
||||
} else {
|
||||
baseName = fmt.Sprintf("section-image-%d", i+1)
|
||||
s.SetAttr("alt", fmt.Sprintf("Section Image %d", i+1))
|
||||
}
|
||||
} else {
|
||||
s.SetAttr("alt", strings.ReplaceAll(baseNameStr, "-", " "))
|
||||
}
|
||||
}
|
||||
|
||||
hash := sha256.Sum256([]byte(imgURL.String()))
|
||||
hashStr := hex.EncodeToString(hash[:])[:6]
|
||||
|
||||
ext := filepath.Ext(imgURL.Path)
|
||||
if ext == "" {
|
||||
ext = ".jpg"
|
||||
}
|
||||
ext = strings.Split(ext, "?")[0]
|
||||
|
||||
// Format output like `capo-notation-a1b2c3.jpg` (using standard url-safe slugs)
|
||||
filename := fmt.Sprintf("%s-%s%s", baseName, hashStr, ext)
|
||||
localAbsPath := filepath.Join(outDir, filename)
|
||||
|
||||
s.SetAttr("src", filename)
|
||||
|
||||
imageJobs = append(imageJobs, ImageJob{
|
||||
URL: imgURL.String(),
|
||||
LocalPath: localAbsPath,
|
||||
})
|
||||
})
|
||||
|
||||
modifiedHTML, err := doc.Find("body").Html()
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("failed to render modified HTML: %w", err)
|
||||
}
|
||||
|
||||
return modifiedHTML, imageJobs, nil
|
||||
}
|
||||
|
||||
// downloadImage fetches the image from the given URL and saves it to localPath.
|
||||
func downloadImage(ctx context.Context, client *http.Client, urlStr, localPath string) (err error) {
|
||||
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
|
||||
if reqErr != nil {
|
||||
return fmt.Errorf("create request for %s: %w", urlStr, reqErr)
|
||||
}
|
||||
|
||||
resp, doErr := client.Do(req)
|
||||
if doErr != nil {
|
||||
return fmt.Errorf("get %s: %w", urlStr, doErr)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("bad status %d for %s", resp.StatusCode, urlStr)
|
||||
}
|
||||
|
||||
out, createErr := os.Create(localPath)
|
||||
if createErr != nil {
|
||||
return fmt.Errorf("create file %s: %w", localPath, createErr)
|
||||
}
|
||||
defer func() {
|
||||
closeErr := out.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err = io.Copy(out, resp.Body); err != nil {
|
||||
return fmt.Errorf("write file %s: %w", localPath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// generateSlug truncates the string to 5 words and converts it to a safe filesystem slug.
|
||||
func generateSlug(s string) string {
|
||||
words := strings.Fields(s)
|
||||
if len(words) > 5 {
|
||||
words = words[:5]
|
||||
}
|
||||
s = strings.Join(words, " ")
|
||||
|
||||
return strings.Trim(slugRegex.ReplaceAllString(strings.ToLower(s), "-"), "-")
|
||||
}
|
||||
Reference in New Issue
Block a user