// ABOUTME: Parses ChatGPT export archives (conversations-*.json) // ABOUTME: into structured session data with DAG linearization // ABOUTME: and content assembly for text, code, thinking, and tools. package parser import ( "encoding/json" "fmt" "os" "path/filepath" "sort" "strings" "time" ) // AssetResolver resolves and copies image assets from exports. type AssetResolver interface { Resolve(pointer string) (path string, ok bool) Copy(srcPath string) (assetRef string, err error) } type chatGPTConversation struct { ID string `json:"conversation_id"` RawID string `json:"id"` Title string `json:"title"` CreateTime *float64 `json:"create_time"` UpdateTime *float64 `json:"update_time"` CurrentNode string `json:"current_node"` Mapping map[string]chatGPTNode `json:"mapping"` } type chatGPTNode struct { ID string `json:"id"` Parent *string `json:"parent"` Children []string `json:"children"` Message *chatGPTMessage `json:"message"` } type chatGPTMessage struct { ID string `json:"id"` CreateTime *float64 `json:"create_time"` Author chatGPTAuthor `json:"author"` Content chatGPTContent `json:"content"` Status string `json:"status"` Metadata chatGPTMeta `json:"metadata"` } type chatGPTAuthor struct { Role string `json:"role"` Name *string `json:"name"` } type chatGPTContent struct { ContentType string `json:"content_type"` Parts []json.RawMessage `json:"parts"` Text string `json:"text"` Language string `json:"language"` Thoughts []chatGPTThought `json:"thoughts"` Title string `json:"title"` URL string `json:"url"` } type chatGPTThought struct { Content string `json:"content"` } type chatGPTMeta struct { ModelSlug string `json:"model_slug"` } // ChatGPTExportParser is implemented by the ChatGPT import-only provider to // stream a ChatGPT data export. ChatGPT sessions are never discovered or // synced from disk; they only enter the archive through a one-shot import, so // this entry point lives on the provider rather than the Discover/Parse path. // Callers obtain it via NewProvider(AgentChatGPT, ...) and a type assertion. type ChatGPTExportParser interface { // ParseChatGPTExport reads all conversations-*.json files from dir and // calls onConversation for each non-empty conversation. ParseChatGPTExport( dir string, assets AssetResolver, onConversation func(ParseResult) error, ) error } // ParseChatGPTExport reads all conversations-*.json files from dir // and calls onConversation for each non-empty conversation. func (p *chatGPTImportOnlyProvider) ParseChatGPTExport( dir string, assets AssetResolver, onConversation func(ParseResult) error, ) error { // Support both numbered shards (conversations-000.json) // and a single conversations.json file. matches, err := filepath.Glob( filepath.Join(dir, "conversations-*.json"), ) if err != nil { return fmt.Errorf("globbing conversations: %w", err) } single := filepath.Join(dir, "conversations.json") if len(matches) == 0 { if _, err := os.Stat(single); err == nil { matches = []string{single} } } if len(matches) == 0 { return fmt.Errorf( "no conversation files found in %s", dir, ) } sort.Strings(matches) for _, path := range matches { if err := parseChatGPTFile( path, assets, onConversation, ); err != nil { return fmt.Errorf("parsing %s: %w", path, err) } } return nil } func parseChatGPTFile( path string, assets AssetResolver, onConversation func(ParseResult) error, ) error { data, err := os.ReadFile(path) if err != nil { return fmt.Errorf("reading file: %w", err) } var convs []chatGPTConversation if err := json.Unmarshal(data, &convs); err != nil { return fmt.Errorf("decoding JSON: %w", err) } for _, conv := range convs { if len(conv.Mapping) == 0 { continue } result, ok := convertChatGPTConversation(conv, assets) if !ok { continue } if err := onConversation(result); err != nil { return err } } return nil } func convertChatGPTConversation( conv chatGPTConversation, assets AssetResolver, ) (ParseResult, bool) { // Fall back to id when conversation_id is empty. convID := conv.ID if convID == "" { convID = conv.RawID } if convID == "" { return ParseResult{}, false } nodes := linearizeDAG(conv.Mapping, conv.CurrentNode) msgs, model := buildChatGPTMessages(nodes, assets) if len(msgs) == 0 { return ParseResult{}, false } var ( userCount int firstMsg string ) for _, m := range msgs { if m.Role == RoleUser && !m.IsSystem { userCount++ if firstMsg == "" { firstMsg = m.Content } } } // Backfill model onto assistant messages that lack one. if model != "" { for i := range msgs { if msgs[i].Role == RoleAssistant && msgs[i].Model == "" { msgs[i].Model = model } } } sess := ParsedSession{ ID: "chatgpt:" + convID, Project: "chatgpt.com", Machine: "local", Agent: AgentChatGPT, FirstMessage: firstMsg, SessionName: conv.Title, StartedAt: unixFloatToTime(conv.CreateTime), EndedAt: unixFloatToTime(conv.UpdateTime), MessageCount: len(msgs), UserMessageCount: userCount, } return ParseResult{Session: sess, Messages: msgs}, true } // linearizeDAG walks parent pointers from currentNode to root, // then reverses to get chronological order. func linearizeDAG( mapping map[string]chatGPTNode, currentNode string, ) []chatGPTNode { if currentNode == "" { return nil } chain := make([]chatGPTNode, 0) nodeID := currentNode for { node, ok := mapping[nodeID] if !ok { break } chain = append(chain, node) if node.Parent == nil || *node.Parent == "" { break } nodeID = *node.Parent } // Reverse to chronological order. for i, j := 0, len(chain)-1; i < j; i, j = i+1, j-1 { chain[i], chain[j] = chain[j], chain[i] } return chain } // buildChatGPTMessages walks linearized nodes and produces // ParsedMessages. Tool nodes are attached to the preceding // assistant message. Returns messages and the last model seen. func buildChatGPTMessages( nodes []chatGPTNode, assets AssetResolver, ) ([]ParsedMessage, string) { var ( msgs []ParsedMessage lastModel string lastAsstID = -1 ordinal int ) for _, node := range nodes { if node.Message == nil { continue } msg := node.Message role := msg.Author.Role content := assembleContent(msg.Content, assets) if msg.Metadata.ModelSlug != "" { lastModel = msg.Metadata.ModelSlug } switch role { case "user": pm := ParsedMessage{ Ordinal: ordinal, Role: RoleUser, Content: content, Timestamp: unixFloatToTime(msg.CreateTime), ContentLength: len(content), } msgs = append(msgs, pm) lastAsstID = -1 ordinal++ case "assistant": hasThinking := msg.Content.ContentType == "thoughts" pm := ParsedMessage{ Ordinal: ordinal, Role: RoleAssistant, Content: content, Timestamp: unixFloatToTime(msg.CreateTime), HasThinking: hasThinking, ContentLength: len(content), Model: msg.Metadata.ModelSlug, } msgs = append(msgs, pm) lastAsstID = len(msgs) - 1 ordinal++ case "system": pm := ParsedMessage{ Ordinal: ordinal, Role: RoleAssistant, Content: content, Timestamp: unixFloatToTime(msg.CreateTime), IsSystem: true, ContentLength: len(content), } msgs = append(msgs, pm) lastAsstID = -1 ordinal++ case "tool": attachToolToAssistant( &msgs, lastAsstID, msg, content, ) } } return msgs, lastModel } // attachToolToAssistant attaches a tool node's content to the // preceding assistant message as a ParsedToolCall. func attachToolToAssistant( msgs *[]ParsedMessage, asstIdx int, msg *chatGPTMessage, content string, ) { if asstIdx < 0 || asstIdx >= len(*msgs) { return } ct := msg.Content.ContentType asst := &(*msgs)[asstIdx] asst.HasToolUse = true switch ct { case "code": asst.ToolCalls = append(asst.ToolCalls, ParsedToolCall{ ToolName: "code_interpreter", Category: NormalizeToolCategory("code_interpreter"), }) case "execution_output": // Pair with the last code_interpreter tool call. if n := len(asst.ToolCalls); n > 0 { tc := &asst.ToolCalls[n-1] tc.ResultEvents = append( tc.ResultEvents, ParsedToolResultEvent{Content: content}, ) } case "tether_quote", "tether_browsing_display": asst.ToolCalls = append(asst.ToolCalls, ParsedToolCall{ ToolName: "web_search", Category: NormalizeToolCategory("web_search"), }) case "multimodal_text": // DALL-E tool responses contain generated images as // multimodal_text. Append the resolved image content // to the assistant message. if content != "" { if asst.Content != "" { asst.Content += "\n\n" } asst.Content += content asst.ContentLength = len(asst.Content) } } } // assembleContent converts a chatGPTContent into a plain-text // string suitable for display. Unicode normalization is applied // only to prose content (text, thoughts, quotes, errors), not // to code blocks, execution output, or URLs. func assembleContent( c chatGPTContent, assets AssetResolver, ) string { switch c.ContentType { case "text", "multimodal_text": return assembleTextParts(c.Parts, assets) case "code": return "```" + c.Language + "\n" + c.Text + "\n```" case "execution_output": return "```\n" + c.Text + "\n```" case "thoughts": var parts []string for _, t := range c.Thoughts { if t.Content != "" { parts = append(parts, normalizeUnicode(t.Content)) } } if len(parts) == 0 { return "" } return "[Thinking]\n" + strings.Join(parts, "\n") + "\n[/Thinking]" case "tether_quote": line := "> " + normalizeUnicode(c.Text) if c.Title != "" || c.URL != "" { line += "\n> -- [" + normalizeUnicode(c.Title) + "](" + c.URL + ")" } return line case "reasoning_recap", "tether_browsing_display": return "" case "system_error": return normalizeUnicode(c.Text) default: return assembleFallbackParts(c.Parts) } } // assembleTextParts joins Parts from text/multimodal_text // content. Each part is either a JSON string or an object. func assembleTextParts( parts []json.RawMessage, assets AssetResolver, ) string { var out []string for _, raw := range parts { var s string if err := json.Unmarshal(raw, &s); err == nil { if s != "" { out = append(out, normalizeUnicode(s)) } continue } // Try as an object with content_type field. var obj struct { ContentType string `json:"content_type"` AssetPointer string `json:"asset_pointer"` } if err := json.Unmarshal(raw, &obj); err != nil { continue } if obj.ContentType == "image_asset_pointer" { out = append(out, resolveImageAsset( obj.AssetPointer, assets, )) } } return strings.Join(out, "\n") } // assembleFallbackParts tries to join parts as strings for // unknown content types. func assembleFallbackParts(parts []json.RawMessage) string { var out []string for _, raw := range parts { var s string if err := json.Unmarshal(raw, &s); err == nil { if s != "" { out = append(out, s) } } } return strings.Join(out, "\n") } // resolveImageAsset attempts to resolve and copy an image asset. func resolveImageAsset( pointer string, assets AssetResolver, ) string { if assets == nil { return "[image unavailable]" } srcPath, ok := assets.Resolve(pointer) if !ok { return "[image unavailable]" } ref, err := assets.Copy(srcPath) if err != nil { return "[image unavailable]" } return "![image](" + ref + ")" } // normalizeUnicode replaces decorative Unicode characters with // their plain ASCII equivalents. var unicodeReplacer = strings.NewReplacer( "\u275D", `"`, // HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT "\u275E", `"`, // HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT "\u275B", "'", // HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT "\u275C", "'", // HEAVY SINGLE COMMA QUOTATION MARK ORNAMENT "\u201C", `"`, // LEFT DOUBLE QUOTATION MARK "\u201D", `"`, // RIGHT DOUBLE QUOTATION MARK "\u2018", "'", // LEFT SINGLE QUOTATION MARK "\u2019", "'", // RIGHT SINGLE QUOTATION MARK "\u2013", "-", // EN DASH "\u2014", "--", // EM DASH "\u2026", "...", // HORIZONTAL ELLIPSIS ) func normalizeUnicode(s string) string { return unicodeReplacer.Replace(s) } func unixFloatToTime(f *float64) time.Time { if f == nil || *f == 0 { return time.Time{} } sec := int64(*f) nsec := int64((*f - float64(sec)) * 1e9) return time.Unix(sec, nsec) }