Add markdown

module etsi.org/standard/tools/docx2md

go 1.23.0

// docx2md converts a TTCN-3 working-document chapter to the Markdown format

// used by ../../src.

package main

import (

	"archive/zip"

	"bytes"

	"encoding/xml"

	"flag"

	"fmt"

	"io"

	"os"

	"regexp"

	"strings"

)

const wordNS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"

var (

	output = flag.String("o", "", "write Markdown to this file instead of standard output")

	plain  = flag.Bool("plain", false, "omit Markdown formatting inferred from Word run formatting")

)

type paragraph struct {

	Properties paragraphProperties `xml:"pPr"`

	Runs       []run               `xml:"r"`

	Hyperlinks []hyperlink         `xml:"hyperlink"`

}

type paragraphProperties struct {

	Style     styleRef  `xml:"pStyle"`

	Numbering numbering `xml:"numPr"`

}

type styleRef struct {

	Value string `xml:"val,attr"`

}

type numbering struct {

	Level numberRef `xml:"ilvl"`

	ID    numberRef `xml:"numId"`

}

type numberRef struct {

	Value string `xml:"val,attr"`

}

type hyperlink struct {

	Runs []run `xml:"r"`

}

type run struct {

	Properties runProperties `xml:"rPr"`

	Text       []text        `xml:"t"`

	Tabs       []struct{}    `xml:"tab"`

	Breaks     []struct{}    `xml:"br"`

}

type text struct {

	Value string `xml:",chardata"`

}

type runProperties struct {

	Bold   *struct{} `xml:"b"`

	Italic *struct{} `xml:"i"`

	Style  styleRef  `xml:"rStyle"`

	Fonts  runFonts  `xml:"rFonts"`

}

type runFonts struct {

	ASCII string `xml:"ascii,attr"`

	HAnsi string `xml:"hAnsi,attr"`

}

type wordStyle struct {

	ID       string        `xml:"styleId,attr"`

	Name     styleRef      `xml:"name"`

	BasedOn  styleRef      `xml:"basedOn"`

	RunProps runProperties `xml:"rPr"`

}

type table struct {

	Rows []tableRow `xml:"tr"`

}

type tableRow struct {

	Cells []tableCell `xml:"tc"`

}

type tableCell struct {

	Paragraphs []paragraph `xml:"p"`

}

func main() {

	flag.Usage = func() {

		fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [-o output.md] [-plain] input.docx\n", os.Args[0])

		flag.PrintDefaults()

	}

	flag.Parse()

	if flag.NArg() != 1 {

		flag.Usage()

		os.Exit(2)

	}

	xmlData, stylesXML, err := wordDocument(flag.Arg(0))

	if err != nil {

		fatal(err)

	}

	markdown, err := convertWithStyles(xmlData, monoStyleIDs(stylesXML))

	if err != nil {

		fatal(err)

	}

	if *output == "" {

		_, err = os.Stdout.WriteString(markdown)

	} else {

		err = os.WriteFile(*output, []byte(markdown), 0o644)

	}

	if err != nil {

		fatal(err)

	}

}

func wordDocument(path string) (document, styles []byte, err error) {

	zr, err := zip.OpenReader(path)

	if err != nil {

		return nil, nil, fmt.Errorf("open %s: %w", path, err)

	}

	defer zr.Close()

	for _, f := range zr.File {

		switch f.Name {

		case "word/document.xml", "word/styles.xml":

		default:

			continue

		}

		r, err := f.Open()

		if err != nil {

			return nil, nil, fmt.Errorf("open %s: %w", f.Name, err)

		}

		data, readErr := io.ReadAll(r)

		r.Close()

		if readErr != nil {

			return nil, nil, fmt.Errorf("read %s: %w", f.Name, readErr)

		}

		if f.Name == "word/document.xml" {

			document = data

		} else {

			styles = data

		}

	}

	if document == nil {

		return nil, nil, fmt.Errorf("%s is not a DOCX file (word/document.xml is missing)", path)

	}

	return document, styles, nil

}

func convert(data []byte) (string, error) {

	return convertWithStyles(data, nil)

}

func convertWithStyles(data []byte, monoStyles map[string]struct{}) (string, error) {

	decoder := xml.NewDecoder(bytes.NewReader(data))

	var blocks []string

	var code strings.Builder

	var inBody bool

	flushCode := func() {

		if code.Len() == 0 {

			return

		}

		text := code.String()

		language := "ttcn3"

		if strings.Contains(text, "::=") {

			language = "ebnf"

		}

		blocks = append(blocks, "```"+language+"\n"+text+"\n```")

		code.Reset()

	}

	for {

		tok, err := decoder.Token()

		if err == io.EOF {

			break

		}

		if err != nil {

			return "", fmt.Errorf("decode document.xml: %w", err)

		}

		start, ok := tok.(xml.StartElement)

		if !ok {

			continue

		}

		if start.Name.Space == wordNS && start.Name.Local == "body" {

			inBody = true

			continue

		}

		if !inBody || start.Name.Space != wordNS {

			continue

		}

		switch start.Name.Local {

		case "p":

			var p paragraph

			if err := decoder.DecodeElement(&p, &start); err != nil {

				return "", err

			}

			kind, text := markdownParagraph(p, monoStyles)

			if text == "" {

				continue

			}

			if kind == "code" {

				if code.Len() > 0 {

					code.WriteByte('\n')

				}

				code.WriteString(text)

			} else {

				flushCode()

				blocks = append(blocks, text)

			}

		case "tbl":

			var tbl table

			if err := decoder.DecodeElement(&tbl, &start); err != nil {

				return "", err

			}

			flushCode()

			if text := markdownTable(tbl, monoStyles); text != "" {

				blocks = append(blocks, text)

			}

		}

	}

	flushCode()

	return strings.Join(blocks, "\n\n") + "\n", nil

}

func markdownParagraph(p paragraph, monoStyles map[string]struct{}) (kind, markdown string) {

	style := strings.ToLower(p.Properties.Style.Value)

	text := strings.TrimSpace(runsMarkdown(p.Runs, monoStyles))

	for _, link := range p.Hyperlinks {

		text += runsMarkdown(link.Runs, monoStyles)

	}

	if text == "" {

		return "", ""

	}

	if level := headingLevel(style); level > 0 {

		return "heading", strings.Repeat("#", level) + " " + withoutClauseNumber(text)

	}

	if isCodeStyle(style) {

		return "code", strings.TrimSpace(runsPlain(p.Runs))

	}

	if p.Properties.Numbering.ID.Value != "" {

		indent := strings.Repeat("  ", number(p.Properties.Numbering.Level.Value))

		if p.Properties.Numbering.ID.Value == "0" {

			return "paragraph", indent + "- " + text

		}

		return "paragraph", indent + "- " + text

	}

	if style == "no" || strings.HasPrefix(strings.ToUpper(text), "NOTE:") {

		return "paragraph", "> " + text

	}

	return "paragraph", text

}

var clauseNumber = regexp.MustCompile(`^\d+(?:\.\d+)*\.?\s+`)

func withoutClauseNumber(text string) string {

	return clauseNumber.ReplaceAllString(text, "")

}

var headingStyle = regexp.MustCompile(`(?:heading|h)[ _-]?([1-9])$`)

func headingLevel(style string) int {

	m := headingStyle.FindStringSubmatch(style)

	if len(m) != 2 {

		return 0

	}

	return int(m[1][0] - '0')

}

func isCodeStyle(style string) bool {

	switch style {

	case "pl", "code", "codeblock", "preformatted", "grammar":

		return true

	}

	return false

}

// monoStyleIDs returns style IDs whose resolved font is monospace or whose

// style is PL. Runs reference these IDs without repeating the font.

func monoStyleIDs(stylesXML []byte) map[string]struct{} {

	if len(stylesXML) == 0 {

		return nil

	}

	var doc struct {

		Styles []wordStyle `xml:"style"`

	}

	if err := xml.Unmarshal(stylesXML, &doc); err != nil {

		return nil

	}

	byID := make(map[string]wordStyle, len(doc.Styles))

	for _, st := range doc.Styles {

		byID[st.ID] = st

	}

	mono := make(map[string]struct{})

	for id := range byID {

		if styleIsMono(id, byID, map[string]bool{}) {

			mono[id] = struct{}{}

		}

	}

	return mono

}

func styleIsMono(id string, styles map[string]wordStyle, seen map[string]bool) bool {

	if id == "" || seen[id] {

		return false

	}

	seen[id] = true

	st, ok := styles[id]

	if !ok {

		return isPLName(id)

	}

	if isPLName(id) || isPLName(st.Name.Value) {

		return true

	}

	if font := latinFont(st.RunProps.Fonts); font != "" {

		return isMonoFont(font)

	}

	return styleIsMono(st.BasedOn.Value, styles, seen)

}

func latinFont(f runFonts) string {

	if f.ASCII != "" {

		return f.ASCII

	}

	return f.HAnsi

}

func isPLName(name string) bool {

	switch strings.ToLower(strings.TrimSpace(name)) {

	case "pl", "plchar", "pl char":

		return true

	default:

		return false

	}

}

func isMonoFont(name string) bool {

	n := strings.ToLower(strings.TrimSpace(name))

	switch {

	case n == "":

		return false

	case strings.Contains(n, "courier"),

		strings.Contains(n, "mono"),

		strings.Contains(n, "typewriter"),

		strings.Contains(n, "consolas"),

		strings.Contains(n, "cascadia"),

		n == "lucida console",

		n == "monaco",

		n == "menlo",

		n == "inconsolata":

		return true

	default:

		return false

	}

}

func runIsCode(r run, monoStyles map[string]struct{}) bool {

	if isPLName(r.Properties.Style.Value) {

		return true

	}

	if _, ok := monoStyles[r.Properties.Style.Value]; ok {

		return true

	}

	return isMonoFont(latinFont(r.Properties.Fonts))

}

func runsMarkdown(runs []run, monoStyles map[string]struct{}) string {

	var b strings.Builder

	var code strings.Builder

	flushCode := func() {

		raw := code.String()

		code.Reset()

		if raw == "" {

			return

		}

		if strings.TrimSpace(raw) == "" {

			b.WriteString(raw)

			return

		}

		left := 0

		for left < len(raw) && (raw[left] == ' ' || raw[left] == '\t') {

			left++

		}

		right := len(raw)

		for right > left && (raw[right-1] == ' ' || raw[right-1] == '\t') {

			right--

		}

		b.WriteString(raw[:left])

		b.WriteByte('`')

		b.WriteString(strings.ReplaceAll(raw[left:right], "`", "\\`"))

		b.WriteByte('`')

		b.WriteString(raw[right:])

	}

	for _, r := range runs {

		s := runText(r)

		if s == "" {

			continue

		}

		if !*plain && runIsCode(r, monoStyles) {

			code.WriteString(s)

			continue

		}

		flushCode()

		b.WriteString(formatRun(s, r))

	}

	flushCode()

	return b.String()

}

func formatRun(s string, r run) string {

	if *plain {

		return s

	}

	switch {

	case r.Properties.Bold != nil && r.Properties.Italic != nil:

		return "***" + s + "***"

	case r.Properties.Bold != nil:

		return "**" + s + "**"

	case r.Properties.Italic != nil:

		return "_" + s + "_"

	default:

		return s

	}

}

func runText(r run) string {

	var text strings.Builder

	for _, t := range r.Text {

		text.WriteString(t.Value)

	}

	for range r.Tabs {

		text.WriteByte('\t')

	}

	for range r.Breaks {

		text.WriteString("  \n")

	}

	return text.String()

}

func runsPlain(runs []run) string {

	var b strings.Builder

	for _, r := range runs {

		text := runText(r)

		text = strings.ReplaceAll(text, "  \n", "\n")

		b.WriteString(text)

	}

	return b.String()

}

func markdownTable(t table, monoStyles map[string]struct{}) string {

	var rows [][]string

	for _, row := range t.Rows {

		var cells []string

		for _, cell := range row.Cells {

			var parts []string

			for _, p := range cell.Paragraphs {

				_, text := markdownParagraph(p, monoStyles)

				if text != "" {

					parts = append(parts, strings.ReplaceAll(text, "|", "\\|"))

				}

			}

			cells = append(cells, strings.Join(parts, "<br>"))

		}

		if len(cells) > 0 {

			rows = append(rows, cells)

		}

	}

	if len(rows) == 0 {

		return ""

	}

	var b strings.Builder

	writeTableRow(&b, rows[0])

	b.WriteByte('\n')

	writeTableRow(&b, make([]string, len(rows[0])))

	b.WriteByte('\n')

	for _, row := range rows[1:] {

		for len(row) < len(rows[0]) {

			row = append(row, "")

		}

		writeTableRow(&b, row[:len(rows[0])])

		b.WriteByte('\n')

	}

	return strings.TrimSpace(b.String())

}

func writeTableRow(b *strings.Builder, cells []string) {

	b.WriteString("|")

	for _, cell := range cells {

		if cell == "" {

			b.WriteString(" --- |")

		} else {

			b.WriteString(" " + cell + " |")

		}

	}

}

func number(s string) int {

	var n int

	for _, c := range s {

		if c < '0' || c > '9' {

			return 0

		}

		n = n*10 + int(c-'0')

	}

	return n

}

func fatal(err error) {

	fmt.Fprintln(os.Stderr, "docx2md:", err)

	os.Exit(1)

}

package main

import (

	"strings"

	"testing"

)

func TestConvert(t *testing.T) {

	const document = `<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>

<w:p><w:pPr><w:pStyle w:val="Heading1"/></w:pPr><w:r><w:t>5 Concepts</w:t></w:r></w:p>

<w:p><w:r><w:t>Ordinary text.</w:t></w:r></w:p>

<w:p><w:pPr><w:pStyle w:val="NO"/></w:pPr><w:r><w:t>NOTE: A note.</w:t></w:r></w:p>

<w:p><w:r><w:t>Use </w:t></w:r><w:r><w:rPr><w:rFonts w:ascii="Courier New" w:hAnsi="Courier New"/><w:b/></w:rPr><w:t>runs</w:t></w:r><w:r><w:rPr><w:rFonts w:ascii="Courier New" w:hAnsi="Courier New"/><w:b/></w:rPr><w:t xml:space="preserve"> on</w:t></w:r><w:r><w:t> here.</w:t></w:r></w:p>

<w:p><w:r><w:rPr><w:rStyle w:val="PLChar"/><w:b/></w:rPr><w:t>altstep</w:t></w:r></w:p>

<w:p><w:r><w:rPr><w:b/></w:rPr><w:t>bold term</w:t></w:r></w:p>

<w:p><w:pPr><w:pStyle w:val="PL"/></w:pPr><w:r><w:t>Identifier ::= "A" .</w:t></w:r></w:p>

<w:tbl><w:tr><w:tc><w:p><w:r><w:t>Column</w:t></w:r></w:p></w:tc></w:tr><w:tr><w:tc><w:p><w:r><w:t>Value</w:t></w:r></w:p></w:tc></w:tr></w:tbl>

</w:body></w:document>`

	got, err := convert([]byte(document))

	if err != nil {

		t.Fatal(err)

	}

	for _, want := range []string{

		"# Concepts",

		"Ordinary text.",

		"> NOTE: A note.",

		"Use `runs on` here.",

		"`altstep`",

		"**bold term**",

		"```ebnf\nIdentifier ::= \"A\" .\n```",

		"| Column |\n| --- |\n| Value |",

	} {

		if !strings.Contains(got, want) {

			t.Errorf("converted Markdown does not contain %q:\n%s", want, got)

		}

	}

}

func TestStyledMonospaceRun(t *testing.T) {

	const styles = `<w:styles xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">

<w:style w:styleId="HTMLCode"><w:name w:val="HTML Code"/><w:rPr><w:rFonts w:ascii="Courier New" w:hAnsi="Courier New"/></w:rPr></w:style>

</w:styles>`

	const document = `<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>

<w:p><w:r><w:rPr><w:rStyle w:val="HTMLCode"/><w:b/></w:rPr><w:t>keyword</w:t></w:r></w:p>

</w:body></w:document>`

	got, err := convertWithStyles([]byte(document), monoStyleIDs([]byte(styles)))

	if err != nil {

		t.Fatal(err)

	}

	if !strings.Contains(got, "`keyword`") {

		t.Fatalf("got %q", got)

	}

	if strings.Contains(got, "**") {

		t.Fatalf("bold monospace span kept bold markup: %q", got)

	}

}

module tools

go 1.26.9

require (

	github.com/yuin/goldmark v1.8.6

	golang.org/x/text v0.43.0

	golang.org/x/tools v0.52.0

)

github.com/yuin/goldmark v1.8.6 h1:d0VcaP1sx9GkFVkoW+KtggpGi2KZ965i14b0+bDQST4=

github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=

golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=

golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=

golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=

golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=