// Package main provides a CLI utility to download HTML articles and convert them to Markdown with localized images. package main import ( "bytes" "context" "crypto/sha256" "encoding/hex" "flag" "fmt" "io" "net/http" "net/url" "os" "os/signal" "path/filepath" "regexp" "strings" "syscall" "time" "codeberg.org/readeck/go-readability/v2" md "github.com/JohannesKaufmann/html-to-markdown" "github.com/PuerkitoBio/goquery" "golang.org/x/sync/errgroup" ) const ( dirPerms = 0755 filePerms = 0644 ) var ( // slugRegex compiled once at package level for performance. slugRegex = regexp.MustCompile("[^a-z0-9]+") ) // ImageJob represents a single image to be downloaded concurrently. type ImageJob struct { URL string LocalPath string } func main() { var titleOverride string var outDirBase string flag.StringVar(&titleOverride, "title", "", "Override the article title and folder name") flag.StringVar(&outDirBase, "out", ".", "Base directory to download the article folder into") flag.Parse() if flag.NArg() < 1 { fmt.Fprintf(os.Stderr, "Usage: %s [options] \nOptions:\n", os.Args[0]) flag.PrintDefaults() os.Exit(1) } rawURL := flag.Arg(0) // Handle graceful shutdown via signals. ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() client := &http.Client{Timeout: 30 * time.Second} if err := run(ctx, client, rawURL, titleOverride, outDirBase); err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) } } // run orchestrates the fetching, parsing, and downloading process. func run(ctx context.Context, client *http.Client, rawURL, titleOverride, outDirBase string) error { targetURL, err := url.Parse(rawURL) if err != nil { return fmt.Errorf("failed to parse target URL %q: %w", rawURL, err) } fmt.Printf("Fetching: %s\n", rawURL) req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) } resp, err := client.Do(req) if err != nil { return fmt.Errorf("failed to fetch URL: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("failed to fetch URL: status code %d", resp.StatusCode) } article, err := readability.FromReader(resp.Body, targetURL) if err != nil { return fmt.Errorf("failed to parse article: %w", err) } finalTitle := article.Title() if titleOverride != "" { finalTitle = titleOverride } slug := generateSlug(finalTitle) if slug == "" { slug = "article" } outDir := filepath.Join(outDirBase, slug) if err := os.MkdirAll(outDir, dirPerms); err != nil { return fmt.Errorf("failed to create directory %s: %w", outDir, err) } var buf bytes.Buffer if err := article.RenderHTML(&buf); err != nil { return fmt.Errorf("failed to render article HTML: %w", err) } modifiedHTML, imageJobs, err := processHTML(buf.String(), targetURL, outDir) if err != nil { return fmt.Errorf("failed to process HTML: %w", err) } converter := md.NewConverter("", true, nil) markdownContent, err := converter.ConvertString(modifiedHTML) if err != nil { return fmt.Errorf("failed to convert to markdown: %w", err) } frontmatter := fmt.Sprintf("---\ntitle: \"%s\"\nsource: \"%s\"\n---\n\n", finalTitle, rawURL) markdownContent = frontmatter + markdownContent if len(imageJobs) > 0 { fmt.Printf("Downloading %d images concurrently...\n", len(imageJobs)) g, gCtx := errgroup.WithContext(ctx) for _, job := range imageJobs { g.Go(func() error { return downloadImage(gCtx, client, job.URL, job.LocalPath) }) } if err := g.Wait(); err != nil { fmt.Fprintf(os.Stderr, "warning: one or more images failed to download: %v\n", err) } } mdPath := filepath.Join(outDir, "index.md") if err := os.WriteFile(mdPath, []byte(markdownContent), filePerms); err != nil { return fmt.Errorf("failed to write markdown file: %w", err) } fmt.Printf("Successfully saved article to: %s\n", mdPath) return nil } // processHTML extracts images, rewrites their src attributes for a flat structure, // and returns the modified HTML string and a slice of ImageJobs to be downloaded. func processHTML(htmlContent string, targetURL *url.URL, outDir string) (string, []ImageJob, error) { doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) if err != nil { return "", nil, err } var imageJobs []ImageJob doc.Find("img").Each(func(i int, s *goquery.Selection) { src, exists := s.Attr("src") if !exists || src == "" { return } imgURL, err := targetURL.Parse(src) if err != nil { fmt.Fprintf(os.Stderr, "warning: failed to parse image URL %s: %v\n", src, err) return } altText, _ := s.Attr("alt") titleText, _ := s.Attr("title") figCaption := s.ParentsFiltered("figure").Find("figcaption").Text() // Traverse up DOM to find nearest preceding section header var headerText string curr := s for curr.Length() > 0 && !curr.Is("body") { h := curr.PrevAllFiltered("h1, h2, h3, h4, h5, h6").First() if h.Length() > 0 { headerText = h.Text() break } curr = curr.Parent() } rawName := "" if altText != "" { rawName = altText } else if titleText != "" { rawName = titleText } else if figCaption != "" { rawName = figCaption } baseName := "" if rawName != "" { s.SetAttr("alt", strings.TrimSpace(rawName)) // set for markdown converter baseName = generateSlug(rawName) if len(baseName) > 40 { baseName = baseName[:40] baseName = strings.TrimRight(baseName, "-") } } if baseName == "" || baseName == "-" { baseNameStr := strings.TrimSuffix(filepath.Base(imgURL.Path), filepath.Ext(imgURL.Path)) baseName = generateSlug(baseNameStr) // Fallback if filename is CDN hash (long) or empty if len(baseName) >= 20 || baseName == "" || baseName == "-" || baseName == "image" { if headerText != "" { baseName = generateSlug(headerText) s.SetAttr("alt", strings.TrimSpace(headerText)) } else { baseName = fmt.Sprintf("section-image-%d", i+1) s.SetAttr("alt", fmt.Sprintf("Section Image %d", i+1)) } } else { s.SetAttr("alt", strings.ReplaceAll(baseNameStr, "-", " ")) } } hash := sha256.Sum256([]byte(imgURL.String())) hashStr := hex.EncodeToString(hash[:])[:6] ext := filepath.Ext(imgURL.Path) if ext == "" { ext = ".jpg" } ext = strings.Split(ext, "?")[0] // Format output like `capo-notation-a1b2c3.jpg` (using standard url-safe slugs) filename := fmt.Sprintf("%s-%s%s", baseName, hashStr, ext) localAbsPath := filepath.Join(outDir, filename) s.SetAttr("src", filename) imageJobs = append(imageJobs, ImageJob{ URL: imgURL.String(), LocalPath: localAbsPath, }) }) modifiedHTML, err := doc.Find("body").Html() if err != nil { return "", nil, fmt.Errorf("failed to render modified HTML: %w", err) } return modifiedHTML, imageJobs, nil } // downloadImage fetches the image from the given URL and saves it to localPath. func downloadImage(ctx context.Context, client *http.Client, urlStr, localPath string) (err error) { req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil) if reqErr != nil { return fmt.Errorf("create request for %s: %w", urlStr, reqErr) } resp, doErr := client.Do(req) if doErr != nil { return fmt.Errorf("get %s: %w", urlStr, doErr) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad status %d for %s", resp.StatusCode, urlStr) } out, createErr := os.Create(localPath) if createErr != nil { return fmt.Errorf("create file %s: %w", localPath, createErr) } defer func() { closeErr := out.Close() if err == nil { err = closeErr } }() if _, err = io.Copy(out, resp.Body); err != nil { return fmt.Errorf("write file %s: %w", localPath, err) } return nil } // generateSlug truncates the string to 5 words and converts it to a safe filesystem slug. func generateSlug(s string) string { words := strings.Fields(s) if len(words) > 5 { words = words[:5] } s = strings.Join(words, " ") return strings.Trim(slugRegex.ReplaceAllString(strings.ToLower(s), "-"), "-") }