chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1 @@
|
||||
github: zakirullin
|
||||
@@ -0,0 +1,9 @@
|
||||
.env
|
||||
.vscode/**
|
||||
.idea/**
|
||||
**/.DS_Store
|
||||
storage
|
||||
fslog
|
||||
tests/node_modules/
|
||||
tests/test-results/
|
||||
tests/playwright-report/
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# --- build stage ---------------------------------------------------------
|
||||
FROM golang:1.24-alpine AS build
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build \
|
||||
-ldflags "-s -w -X main.Version=${VERSION}" \
|
||||
-o /out/server ./cmd/server
|
||||
|
||||
# --- runtime stage -------------------------------------------------------
|
||||
FROM alpine:3.20
|
||||
RUN apk add --no-cache ca-certificates tzdata && \
|
||||
addgroup -g 1000 app && adduser -D -u 1000 -G app app
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/server /app/server
|
||||
COPY web /app/web
|
||||
|
||||
RUN mkdir -p /app/storage /app/tokens && chown -R app:app /app
|
||||
VOLUME ["/app/storage", "/app/tokens"]
|
||||
|
||||
|
||||
USER app
|
||||
|
||||
ENTRYPOINT ["/app/server"]
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Artem Zakirullin
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,148 @@
|
||||
.PHONY: server docker_build docker_run compose_up compose_down
|
||||
|
||||
server:
|
||||
go run ./cmd/server
|
||||
|
||||
docker_build: # build container image (override with `make docker_build DOCKER=podman`)
|
||||
$(DOCKER) build -t files-md --build-arg VERSION=$$(git rev-parse --short HEAD) .
|
||||
|
||||
docker_run: # run container, host 80 -> container 8080
|
||||
$(DOCKER) run --rm -it -p 80:8080 \
|
||||
-v files-md-storage:/app/storage \
|
||||
-v files-md-tokens:/app/tokens \
|
||||
-e APP_URL=http://localhost \
|
||||
-e STORAGE_DIR=/app/storage \
|
||||
-e TOKENS_DIR=/app/tokens \
|
||||
-e CERT_DIR= \
|
||||
files-md
|
||||
|
||||
compose_up: # build + start via compose.yaml, logs attached
|
||||
$(DOCKER) compose up --build
|
||||
|
||||
compose_down: # stop and remove the compose container (named volumes survive)
|
||||
$(DOCKER) compose down
|
||||
|
||||
DOCKER ?= docker
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
check:
|
||||
go fmt ./... && go vet ./... && go test ./...
|
||||
|
||||
lint:
|
||||
golangci-lint run
|
||||
|
||||
format:
|
||||
gofumpt -w .
|
||||
|
||||
e2e: # make e2e test="create and move"
|
||||
killall server || true
|
||||
go run ./cmd/server > /tmp/server_err 2>&1 & \
|
||||
cd tests && npm run test $(if $(test),-g "$(test)")
|
||||
|
||||
e2eh: # headed e2e tests
|
||||
killall server || true
|
||||
go run ./cmd/server & \
|
||||
cd tests && npm run test:headed $(if $(test),-g "$(test)")
|
||||
|
||||
e2es: # run single test, e2es test="name"
|
||||
cd tests && npm run test -- $(if $(test),-g "$(test)")
|
||||
|
||||
e2esh: # run single test headed, e2esh test="name"
|
||||
cd tests && npm run test:headed -- $(if $(test),-g "$(test)")
|
||||
|
||||
perf:
|
||||
cd tests && PERF=1 npm run test -- perf.spec.js
|
||||
|
||||
sync:
|
||||
killall server || true
|
||||
go run ./cmd/server & \
|
||||
cd tests && npm run test --g "sync"
|
||||
|
||||
synch:
|
||||
killall server || true
|
||||
go run ./cmd/server & \
|
||||
cd tests && npm run test:headed --g "sync"
|
||||
|
||||
report:
|
||||
cd tests && npx playwright show-report
|
||||
|
||||
define ENV_FILE
|
||||
BOT_API_TOKEN=
|
||||
API_HOST=$(apihost)
|
||||
APP_HOST=app.files.md
|
||||
STORAGE_DIR=/app/storage
|
||||
CERT_DIR=/opt/files.md
|
||||
TOKENS_DIR=/opt/files.md/tokens
|
||||
LOG_FILE=/var/log/files.md/server.log
|
||||
endef
|
||||
|
||||
define SERVICE_FILE
|
||||
[Unit]
|
||||
Description=Files.md Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=www-data
|
||||
ExecStart=/app/server
|
||||
WorkingDirectory=/app
|
||||
Environment=TOKENS_SALT=$(salt)
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:/app/log
|
||||
StandardError=append:/app/err
|
||||
AmbientCapabilities=CAP_NET_BIND_SERVICE
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
endef
|
||||
|
||||
# make init_server host=root@1.2.3.4 salt=my-secret-salt apihost=api.example.com
|
||||
export ENV_FILE SERVICE_FILE
|
||||
init_server: # create directories and configuration files on the server
|
||||
ssh $(host) 'sudo mkdir -p /app/storage /var/log/files.md /opt/files.md /opt/files.md/tokens && \
|
||||
sudo chown -R www-data:www-data /app /var/log/files.md /opt/files.md'
|
||||
echo "$$ENV_FILE" | ssh $(host) 'sudo tee /app/.env > /dev/null && sudo chown www-data:www-data /app/.env'
|
||||
echo "$$SERVICE_FILE" | ssh $(host) 'sudo tee /etc/systemd/system/filesmd.service > /dev/null'
|
||||
ssh $(host) 'sudo systemctl daemon-reload && sudo systemctl enable filesmd.service'
|
||||
@echo 'Directories created and permissions set successfully.'
|
||||
|
||||
deploy_systemd: # deploy as systemd service
|
||||
@GREEN='\e[32m'; \
|
||||
YELLOW='\e[33m'; \
|
||||
RESET='\e[0m'; \
|
||||
COMMIT_HASH=$$(git rev-parse --short HEAD); \
|
||||
printf "$${YELLOW}Building (version=$${COMMIT_HASH})...$${RESET}\n" && \
|
||||
make check && \
|
||||
GOOS=linux GOARCH=amd64 go build -ldflags "-X main.Version=$${COMMIT_HASH}" -o /tmp/server ./cmd/server && \
|
||||
printf "$${GREEN}Build Completed$${RESET}\n" && \
|
||||
scp /tmp/server $(host):/tmp/server.new && printf "$${GREEN}The binary is copied on the server$${RESET}\n" && \
|
||||
ssh $(host) "sudo mv /tmp/server.new /app/server && sudo systemctl daemon-reload && sudo systemctl restart filesmd.service" && \
|
||||
rm /tmp/server && \
|
||||
printf "$${YELLOW}Version: $${COMMIT_HASH}$${RESET}\n" && \
|
||||
TMPWEB=$$(mktemp -d) && \
|
||||
cp -r web "$${TMPWEB}/web" && \
|
||||
find "$${TMPWEB}/web" -name "*.html" -exec grep -l "?v=" {} \; | xargs sed -i '' 's/?v=/?v='"$${COMMIT_HASH}"'/g' && \
|
||||
tar --no-xattrs --disable-copyfile --no-fflags -czf "$${TMPWEB}/web.tar.gz" -C "$${TMPWEB}" web && \
|
||||
scp "$${TMPWEB}/web.tar.gz" $(host):/app/ && \
|
||||
ssh $(host) "cd /app && tar -xzf web.tar.gz && rm web.tar.gz" && \
|
||||
rm -rf "$${TMPWEB}" && \
|
||||
printf "$${GREEN}Successfully deployed!$${RESET}\n"
|
||||
|
||||
deploy_binary: # deploy as regular binary, kinda deprecated, but ok for simple setup
|
||||
@GREEN='\e[32m'; \
|
||||
YELLOW='\e[33m'; \
|
||||
RESET='\e[0m'; \
|
||||
COMMIT_HASH=$$(git rev-parse --short HEAD); \
|
||||
printf "$${YELLOW}Building (version=$${COMMIT_HASH})...$${RESET}\n" && \
|
||||
make check && \
|
||||
GOOS=linux GOARCH=amd64 go build -ldflags "-X main.Version=$${COMMIT_HASH}" -o /tmp/server ./cmd/server && \
|
||||
printf "$${GREEN}Build Completed$${RESET}\n" && \
|
||||
ssh $(host) "sudo killall server || true" && \
|
||||
scp /tmp/server $(host):/tmp/server.new && printf "$${GREEN}The binary is copied on the server$${RESET}\n" && \
|
||||
ssh $(host) "sudo mv /tmp/server.new /app/server && sudo setcap 'cap_net_bind_service=+ep' /app/server" && \
|
||||
ssh $(host) "sudo su -c \"cd /app && nohup ./server >> /app/log 2>>/app/err &\" -s /bin/sh www-data" && \
|
||||
rm /tmp/server && \
|
||||
printf "$${GREEN}Successfully deployed!$${RESET}\n"
|
||||
|
||||
@@ -0,0 +1,411 @@
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/icon.png" alt="Files.md icon" title="Files.md" align="right" height="76" />
|
||||
|
||||
# Files.md
|
||||
Private, quiet space for thinking. Simple app for `.md` files.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/appdark.png" alt="Files.md screenshot" title="Files.md"/>
|
||||
English · <a href="README.zh-CN.md">简体中文</a>
|
||||
</p>
|
||||
|
||||
You can store your whole life:
|
||||
- 📌 Notes
|
||||
- 📝 Documents, Projects
|
||||
- 💚 Journal, Habits
|
||||
- ✅ Checklists, Tasks
|
||||
|
||||
All in plain `.md` files, local-first. LLM-friendly. **Private - your data stays at your device**.
|
||||
|
||||
Try it out: [app.files.md](https://app.files.md) (Beta). Main site: [files.md](https://files.md).
|
||||
|
||||
You can use this app for Second Brain, Zettelkasten, Notes, Journaling, Tasks, Checklists, and more.
|
||||
|
||||
> Own your data as plain local files.
|
||||
> Own the software that opens those files.
|
||||
> Grow your personal workspace with files.
|
||||
> Grow the software around it with an LLM.
|
||||
> **Local files and owned software last ages**.
|
||||
|
||||
**I have been building this project for 5 years**. Consider **[supporting on GitHub 💚](https://github.com/sponsors/zakirullin)**.
|
||||
|
||||
[Dump your thoughts](#dump-your-thoughts) · [How to think deeply](#how-to-think-deeply) · [Second Brain?](#second-brain) · [Chatbot](#chatbot) · [Journal, tasks, checklists](#journal) · [Files structure](#files-structure)
|
||||
|
||||
## Another note taking app?
|
||||
Maybe. But this time:
|
||||
- Only necessary features, **restrictions foster creativity**
|
||||
- Free and open source
|
||||
- No need to install anything, all you need is a browser
|
||||
- Works offline
|
||||
- **Local-first, files don't leave your device**
|
||||
- Extremely simple code. **One person or an LLM can fit the whole project in head**
|
||||
- In other words, this project has [low cognitive load](https://github.com/zakirullin/cognitive-load)
|
||||
- The codebase is ready for your LLM to extend to your needs
|
||||
- Portable, no build systems, no Electron, just open `web/index.html`
|
||||
- Optional out of the box synchronization
|
||||
- The server is just one binary (or use iCloud/Dropbox/Google Drive for sync)
|
||||
- Telegram chatbot for on-the-go access to your files
|
||||
- **Chat-like flow for effortless thought capture**
|
||||
|
||||
## How to use
|
||||
- Open [app.files.md](https://app.files.md)
|
||||
- Click "Install files.md" on the right side of the address bar:
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/install.png" alt="Install" title="Install" width="350"/>
|
||||
</div>
|
||||
|
||||
- Open a local folder to persist changes
|
||||
- Occasionally hit force-refresh (`Cmd+Shift+R`) to get new updates.
|
||||
|
||||
P.S. For now, Chromium-based browsers (like Chrome, [Brave](https://github.com/zakirullin/files.md/issues/61)) are best at [File System API support](https://caniuse.com/native-filesystem-api).
|
||||
|
||||
## Dump your thoughts
|
||||
Open the chat (`Cmd+Enter`) and send a message:
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/note1.png" width="350"/>
|
||||
</div>
|
||||
|
||||
Choose where to save (can do later):
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/note2.png" width="350"/>
|
||||
</div>
|
||||
|
||||
**With this flow you can quickly save notes, tasks, journal records and checklists**.
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/app.png" alt="Files.md screenshot" title="Files.md" width="700"/>
|
||||
</div>
|
||||
|
||||
Everything can be synchronized with the chatbot.
|
||||
|
||||
## Chatbot
|
||||
Open the chat, write something and press `Enter`:
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/bot.gif" alt="Saving things in bot" title="Saving things in bot" width="350"/>
|
||||
</div>
|
||||
|
||||
That's it, your note is saved to an `.md` file.
|
||||
|
||||
Full bot's functionality:
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/bot.png" width="700"/>
|
||||
</div>
|
||||
|
||||
Don't worry - it's much simpler than that by default.
|
||||
|
||||
[Telegram Bot](https://t.me/FilesMDBot). *Other messengers will follow...*
|
||||
|
||||
## How to think deeply
|
||||
I used [app.files.md](https://app.files.md) to improve my thinking in brain and software development area.
|
||||
|
||||
How I did that:
|
||||
- I read articles and books
|
||||
- I took notes in the chat, so that my reading flow wasn't interrupted
|
||||
- One idea or insight per note
|
||||
- Each note should be understandable without context
|
||||
- I moved notes to either `brain` or `dev` folders (can be done after reading)
|
||||
|
||||
The main thing at this point is not to distract yourself from the reading process. That's why I use the chat.
|
||||
|
||||
Occasionally, I did the following thought work:
|
||||
- **I spent time travelling through the notes and thinking them through**
|
||||
- I made connections between the relevant notes (type `[`)
|
||||
- Notes should be connected, just like neurons in our brain
|
||||
- **I practiced my new knowledge, I applied it out in the world**
|
||||
- At one point, some ideas in `brain` and `dev` folders appeared very related
|
||||
- This connection between two different domains produced an insight
|
||||
- I wrote an article based on that insight: [Cognitive Load in Software Development](https://github.com/zakirullin/cognitive-load)
|
||||
|
||||
All this activity helped me to:
|
||||
- **Think deeply** (which is very important in the AI-age)
|
||||
- **Think systematically and see the bigger picture**
|
||||
- **Write insightful texts**
|
||||
|
||||
To achieve all that, **you'll have to use your brain**, not advanced templates or AI workflows.
|
||||
|
||||
- Start with no structure at all, 0 folders
|
||||
- One idea per note
|
||||
- Every note should be understood without context
|
||||
- Apply new knowledge immediately, don't save it for future self
|
||||
- Link related notes
|
||||
- **Revisit your notes and think through**
|
||||
|
||||
My friends and I have been using this simple setup for five years, and it works well.
|
||||
|
||||
## Second Brain?
|
||||
I'll quote [I Deleted My Second Brain](https://web.archive.org/web/20260518143828/https://www.joanwestenberg.com/i-deleted-my-second-brain-692aa40d59d5f06dd5131e43/):
|
||||
|
||||
> Obsidian is a brilliant piece of software. I love it, dearly. But like anything, without restraint, it can also be a trap. Markdown files in nested folders. Plugins that track your productivity. Graph views that suggest omniscience. There’s an illusion of mastery in watching your notes web into constellations. But constellations are projections. They tell stories. They do not guarantee understanding.
|
||||
>
|
||||
> When I first started using PKM tools, I believed I was solving a problem of forgetting. Later, I believed I was solving a problem of integration.
|
||||
>
|
||||
> **Eventually, I realized I had created a new problem: deferral. The more my system grew, the more I deferred the work of thought to some future self who would sort, tag, distill, and extract the gold.**
|
||||
>
|
||||
> **That self never arrived.**
|
||||
|
||||
The Second Brain is thrilling.
|
||||
Advanced guru templates, plugins and AI workflows...
|
||||
One wants to scrape the wisdom of the whole internet.
|
||||
There's some beauty in this neat system. Every new note brings dopamine.
|
||||
|
||||
*Second Brain is getting better*.
|
||||
**But the first brain is not improving**.
|
||||
And that’s an issue.
|
||||
In the AI age, your first brain is as valuable as ever.
|
||||
|
||||
Use **your brain** to think through the notes.
|
||||
The tool is not important, your thinking is.
|
||||
|
||||
Before adding a new note, try to answer these questions:
|
||||
- How this new knowledge can sharpen my judgment or expand my taxonomy?
|
||||
- How can I see the world differently, given this new knowledge?
|
||||
- How can I act differently?
|
||||
|
||||
## Notes can prevent experience
|
||||
- Reading and taking notes can easily fool us into believing that we understand a text
|
||||
- We think we understand, but in reality **we just know**
|
||||
- **At some point our "knowing" is so good, that we start feeling that we actually do it (or at least tried)**
|
||||
|
||||
The worst thing is that **we don’t let new experiences emerge because we already have knowledge**. It's a knowledge barrier. Life gives us opportunities to live through new experiences, but we refuse, because "we already know".
|
||||
|
||||
## Self-help through reading and taking notes?
|
||||
**Harm caused at the emotional level must be healed at the emotional level.**
|
||||
|
||||
Not through intellectual work and taking notes.
|
||||
Reading without action is entertainment. A form of procrastination.
|
||||
**No amount of self-help books can heal emotional wounds.**
|
||||
What can help is psychotherapy, rescripting and chair work. Meditation.
|
||||
**Healing happens by feeling.**
|
||||
|
||||
## When to take notes
|
||||
If your goal is to:
|
||||
- Develop a deeper, more structured understanding of something
|
||||
- Do research
|
||||
- Write an article or a book
|
||||
|
||||
Then taking notes is perfectly fine.
|
||||
|
||||
## Journal
|
||||
Are you feeling good about something? Send a message.
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/journal.png" title="Journaling" width="350"/>
|
||||
</div>
|
||||
|
||||
Then click "To Journal".
|
||||
Or just add ` jj` or ` жж` at the end of your message.
|
||||
|
||||
Your record is gonna be saved nicely in the `journal/YYYY.MM Month.md` file.
|
||||
|
||||
## Tasks ✅
|
||||
You're in the flow.
|
||||
A colleague asks you to send a report.
|
||||
Holding that in your head drains energy.
|
||||
Drop a message and stay in flow.
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/task.png" title="Adding a task" width="350"/>
|
||||
</div>
|
||||
|
||||
Add only small, actionable items. Not things like "Plan a vacation".
|
||||
Your task should be the first, small step of what should be done anyway.
|
||||
**Not what you wish to be done, but don't have motivation just yet.**
|
||||
Your task list should not create a feeling of guilt.
|
||||
|
||||
For tasks that should be done later press "To Later".
|
||||
|
||||
## Checklists 🛒
|
||||
A friend recommends a book to you.
|
||||
You're out of butter and buns.
|
||||
Holding things like this in your head is taxing.
|
||||
|
||||
Drop all that to chat, then move to a matching checklist.
|
||||
|
||||
## How to sync
|
||||
| Setup | Where your files live | Sync across devices | Server needed | Best for |
|
||||
|-------------------------------------------------------|---------------------------------------|-------------------|--------------------|------------------------------------------------------|
|
||||
| **Local-first**, `app.files.md` doesn't send any data | A folder on your device | No | None | Maximum privacy, your data doesn't leave your device |
|
||||
| **Cloud-folder sync** (iCloud/Dropbox/Google Drive) | Your existing cloud folder | Yes | None (cloud provider) | Sync across devices without running a server |
|
||||
| **[Self-hosted sync server](docs/your-own-server.md)** | Your own (or local) server | Yes | One Go binary | Sync between devices inside your network |
|
||||
| **Hosted sync server** | Our managed server | Yes | `api.files.md` | Try it instantly, no setup |
|
||||
|
||||
## Files structure
|
||||
You don't have to think about the structure, it is predefined.
|
||||
Although, you're free to use whatever structure you want.
|
||||
|
||||
- Chat: `Chat.md`
|
||||
- Notes: `brain/Note.md`, `<category>/*.md`
|
||||
- Projects: `Project.md`, `*.md`
|
||||
- Checklists: `Read.md`, `Watch.md`, `Shop.md`
|
||||
- Journal: `journal/2024.08 August.md`
|
||||
- Tasks: `Later.md`
|
||||
- Habits: `habits/Ate consciously.md`, `habits/*.md`
|
||||
- Images: `media/*` (png, jpg, webp, gif)
|
||||
- Archive: `archive/*.md`
|
||||
- Config: `config.json`
|
||||
|
||||
Place [AGENTS.md](https://github.com/zakirullin/cognitive-load/blob/main/web/AGENTS.md) in your knowledge base - so that your files become a live personal space AI can work with.
|
||||
|
||||
## Hotkeys
|
||||
|
||||
| Hotkey | Action |
|
||||
|----------------------------------------|------------------------------|
|
||||
| `[` | Insert a link to a file |
|
||||
| `Cmd+K` / `Ctrl+K` | Open file search modal |
|
||||
| `Cmd+N` / `Ctrl+N` | New file |
|
||||
| `Cmd+M` / `Ctrl+M` | Move file |
|
||||
| `Cmd+D` / `Ctrl+D` | Delete file |
|
||||
| `Cmd+Enter` / `Ctrl+Enter` | Open chat |
|
||||
| `Cmd+Shift+Enter` / `Ctrl+Shift+Enter` | Toggle chat dialog |
|
||||
| `Cmd+[` / `Ctrl+[` | Go to previous file |
|
||||
| `Cmd+]` / `Ctrl+]` | Go to next file |
|
||||
| `Cmd+~` / `Ctrl+~` | Toggle sidebar |
|
||||
| `Cmd+B` / `Ctrl+B` | Toggle **bold** |
|
||||
| `Cmd+I` / `Ctrl+I` | Toggle *italic* |
|
||||
| `Cmd+Y` / `Ctrl+Y` | Insert checkbox |
|
||||
| `Cmd+T` / `Ctrl+T` | Insert table |
|
||||
| `Cmd/Ctrl` + `Click` | Copy inline text / open link |
|
||||
| `Ctrl+Cmd+Space` | Insert emoji (macOS) |
|
||||
|
||||
## Useful scripts for your files
|
||||
All scripts are in `cmd` and can be run **inside your files directory**. Install [Go](https://go.dev/doc/install) first.
|
||||
|
||||
### Add Whoop metrics to journal
|
||||
```
|
||||
go run /abs/path/to/files.md/cmd/whoop/whoop.go
|
||||
```
|
||||
|
||||
### Convert wikilinks to markdown links
|
||||
Convert `[[wikilinks]]` to standard `[Name](/path.md)` (`--dry-run` available):
|
||||
```
|
||||
go run /abs/path/to/files.md/cmd/tomdlinks/tomdlinks.go .
|
||||
```
|
||||
|
||||
### Insert backlinks
|
||||
Adds links back to referencing files (`--dry-run` available):
|
||||
```
|
||||
go run /abs/path/to/files.md/cmd/backlink/backlink.go
|
||||
```
|
||||
|
||||
### Shift journal timestamps
|
||||
Shift timestamps in journal files by N hours (useful after timezone change):
|
||||
```
|
||||
go run /abs/path/to/files.md/cmd/shifttime/shifttime.go
|
||||
```
|
||||
|
||||
## Documentation
|
||||
[Deploy on your own server](docs/your-own-server.md)
|
||||
[Chatbot](docs/bot.md)
|
||||
[Sync flow](docs/sync-flow.md)
|
||||
[End-to-end tests](docs/e2e-tests.md)
|
||||
|
||||
## Repository structure
|
||||
- `web` - web app (PWA)
|
||||
- `web/index.html` entrypoint for the web app
|
||||
- `web/lib` - web app libs
|
||||
- `cmd/server` - entrypoint for the server
|
||||
- `cmd/*/` - useful scripts for `.md` files
|
||||
- `server` - server code
|
||||
- `server/bot.go` - chatbot
|
||||
- `server/sync/` - sync API server code
|
||||
- `vendor` - backend libs
|
||||
- `tests` - E2E tests, test both the web app and the server
|
||||
|
||||
## How to contribute
|
||||
- Junior developers should be able to understand the code
|
||||
- Ideally, every PR should remove or simplify code, not add it
|
||||
- **The less code we have, the more flexible we are**
|
||||
- All dependencies are our code and responsibility. So, avoid dependencies if possible
|
||||
- Code should be self-sufficient, so `vendor` and `web/lib` folders are included in the repository
|
||||
- **Do we really need this feature? Will it help us to do the real job, or does it just give dopamine?**
|
||||
|
||||
Refer to [this guide](https://github.com/zakirullin/cognitive-load) for more comprehensive rules.
|
||||
|
||||
## Backend guidelines
|
||||
- We write **tests**
|
||||
- We don't use get* prefix for methods
|
||||
- No panics, errors are part of business logic
|
||||
- If we are ignoring an error - we leave a WHY comment
|
||||
- We wrap errors all the time, we should add method's context
|
||||
- No iterators for client code
|
||||
- We prefer real implementations or at least fakes over mocks and stubs
|
||||
- Imports should only be renamed to avoid a name collision with other imports
|
||||
- **With portability in mind, everything is stored in plain `.md` files**
|
||||
|
||||
## Frontend guidelines
|
||||
- Use `PATCHED` keyword if you modify libs in-place
|
||||
- **It would be fantastic if, one day, we replaced `CodeMirror` with our own tiny implementation**
|
||||
- No build systems, **in 10 years we will open `/web/index.html` and it should just work**
|
||||
- Don't forget that awaits between lock check and lock acquire can cause race condition
|
||||
- Avoid flaky e2e tests. First we get negative emotions, then we stop running all the tests
|
||||
- Most bugs are caused due to race conditions, when an async flow is interrupted mid through
|
||||
|
||||
## Glossary
|
||||
- `filename` - a filename with extension, like "note.md" (USE THIS AS ID)
|
||||
- `header` - an extension-stripped and capitalized filename, like "Note"
|
||||
- `body` - file's content
|
||||
- `dir` - a dir that is meant to store notes under some category, like "happiness"
|
||||
- `userID` - chatID. For the most part we're only using chatID as userID (PM with the bot)
|
||||
- `ctime` for file - data blocks or metadata change time: file's ownership, location, file type and permission settings changed time. Parent folder renaming won't affect, moving the file does affect, renaming the file does affect. We need this to track file's location changes, like to understand when it was moved to archive, to track task's angry level etc
|
||||
- `mtime` for file - mtime (modification time) for a file refers to the time when the contents of the file were last modified. Unlike ctime, it is not affected by changes to the file's metadata, such as ownership, permissions, or renaming. We rely on that for synchronization.
|
||||
- `ctime` for dir - adding or removing files or subdirectories (similar to `mtime` plus inode changes like renaming files)
|
||||
|
||||
Any file can be uniquely identified by filename and dir. We only support one level of nesting.
|
||||
|
||||
## Performance
|
||||
The project is blazing fast :) If you're afraid of using files or mutexes unnecessarily for performance reasons, take a look at this:
|
||||
```
|
||||
Mutex lock/unlock = 25 ns
|
||||
Read 4K randomly from SSD = 150,000 ns
|
||||
1 ms = 1,000,000 ns
|
||||
```
|
||||
|
||||
## ADRs (Architecture Decision Records)
|
||||
- `20.06.2026` Backlinks are now inserted automatically whenever we link a note. For our workspace to be cross-platform (so you can view it even in GitHub), we insert backlinks on the fly, and not infer them dynamically. That way from any viewer backlinks would work.
|
||||
- `01.06.2026` Audio and video are now supported. I believe enriching journal with these kind of media can be useful for our emotional healing.
|
||||
- `24.05.2026` Unfold everything (images, math, mermaid) in viewport immediately. It prevents flickering and doesn't add performance penalty. P.S. We should also schedule full document unfold on viewportChange, otherwise latex/mermaid block off the viewport won't be folded.
|
||||
- `22.05.2026` Added Mermaid support. It was decided to lazy-load the script, because mermaid.min.js size is 3MB. It's quite a load to load synchronously for such a small app.
|
||||
- `20.05.2026` Added LaTeX support, even though I wasn't happy about +20 font files. LaTeX is text-based and LLM-friendly. Text + Math will cover pretty much everything.
|
||||
- `06.05.2026` Moved from Today.md to Chat.md. CustDev showed that users have trouble grasping "today" concept. And besides, "open chat" phrase has meaning in both bot and webapp.
|
||||
- `02.05.2026` Now hide-token runs synchronously on every change, previously it had 100ms debounce which caused jitter on by word removals in links and formatted texts.
|
||||
- `02.05.2026` Merged Inbox.md and Today.md to Today.md. Inbox name is too abstract, productivity-related and GTD-ish. I want calmness and simplicity. Today is like "the page I live in chat".
|
||||
- `23.04.2026` Moved from API_HOST, APP_HOST to API_URL, APP_URL. For different environments it's better to provide more information like desired schema in configuration.
|
||||
- `22.04.2026` Inbox entries in the bot are now identified by a stable content hash (`fs.Hash` of the block with the `- [ ] `/`- [x] ` marker stripped) instead of a positional index, so a button keeps pointing at the right line even if other entries are added/removed/completed in between.
|
||||
- `22.04.2026` It was mentally taxing to see two buttons/messages "to inbox" and "to chat", it was not as mentally easy just to drop a task for chat. Because it went to inbox, and 1 more click needed. That one click was the reason adding new tasks became frustrating. I let go of two different flow, and now everything goes to inbox, and every item is inbox is a markdown checklist item. As a bonus, PWA app is now very handy as it shows tasks for chat by default. Also, maybe "inbox" is a mentally overloaded term, and "chat" sounds better. Will see.
|
||||
- `11.04.2026` Even though I want to store links as plain markdown links, visually I want to work with them as if they were minimal [links]. For that I decided to hide (...) part when cursor is on the line. The (...) part is only hidden for markdown-files link.
|
||||
- `11.04.2026` Brought back standard Markdown Links. I want the knowledge base to be cross-platform. It should work in GitHub.
|
||||
- `05.04.2026` Tried to move web/* stuff in the root folder for simplicity. Bad decision - there should be an explicit dir which we can use as public DOCROOT on our server.
|
||||
- `21.09.2025` Switched to [link] for links. The `[link](full%20path)` syntax is too overwhelming and clunky, plus we don't want to deal with path changes.
|
||||
- `21.09.2025` Removed WASM. I had a bug when a message was removed from Inbox.txt, and was not added to a file (I pressed "move to file" button). I wasn't able to reproduce the issue, but what I found is a lot of complexity. JS -> Go (writeFile) -> Go awaiting a promise from JS -> JS Golang runtime somewhere in between -> JS (writeFile) -> Go (returning from promise) -> Sending results back to JS. And it has to be done in a separate goroutine, because both WASM and JS are running in the same thread. Also, Golang's WASM is still experimental. We have too many components and a lot of uncertainty involved. I didn't want to implement same functionality in JS back then, at the solution served for some time. Now it's time to reimplement the functionality in JS and give up all this complexity. Also, inbox.wasm is ~8MB and I wanted the application to be really small.
|
||||
- `11.07.2025` Decided to use OPFS as an initial driver for file system. Better browsers support, less hustle for users. The app starts with OPFS driver by default, if needed, user can replace the driver with Local FileSystem API by opening a local dir. DirHandle would be saved to IndexedDB in such scenario and reused every time.
|
||||
- `08.07.2025` Root folder is now '/', not ''. All files in webapp are identified by path, not by 'dir' + 'filename', restricting to 1 level of nesting.
|
||||
- `08.07.2025` Dropbox is changing some metadata for newly created files, thus ctime is changed. I was thinking about moving to mtime for sync, but that wouldn't allow us detect renames (though, we detect them through a separate mechanism anyway), so mtime can be more reliable. Also sync won't be triggered by permission/ownership change etc. Migrated to mtime. Mtime is used for content-based sync, ctime is used for append-only sync log (renames/del). Also we can restore mtime from .git/archive, unlike ctime.
|
||||
- `30.06.2025` Decided to migrate every flow to Chat.md, even todo lists. Added - we can't work with multiline tasks with this flow, we may want support both files and indices. We have two ways of doing so - encode params in a uniform way, and use same command handlers with IFs. Or we can use different command handlers to handle chat/file movements. I decided to go for different command handlers. Added, if we go for different commands - move to buttons config would be complicated. Added, maybe we can move files back to Chat.md on "file move", and reuse the existing flow? Added, so far seems good. Our chat.md log acts as an append-only log. As a bonus, if we don't finish some flow (like schedule/move), the content would be saved in log and we can continue scheduling/moving from the app.
|
||||
- `29.06.2025` All incoming messages go to Chat.md now by default. Before that they got moved to `/today` (and become tasks), which was good for a simple todo list, but not as convenient for other use cases. I realized that during meetings, all I needed was a simple input field where I can dump whatever stuff from my head with no further immediate action. With a possibility to review and organize it later. It can be tasks, it can be journal records, or it can be files. Also, it's better to have a really simple easy to understand default flow - we dump all the messages into one file, and that's it.
|
||||
- `27.06.2025` Default mode for chat is "One big file" now, i.e. the only thing it does is dumps all the messages into one file. Again, let's start with the simplest flow, not to overwhelm users. Added later. If we choose full mode, we'll have to create dirs upfront so that "to habits", "to read/shop" etc. would work. If users don't need it, he removes the dirs, and we don't recreate them (as we would do in "on-the-fly mode"). So, we can't use on-the-fly strategy everywhere.
|
||||
- `26.06.2025` Before we created all necessary dirs upfront, now we create dirs on the fly. That way we won't clutter user's knowledge base right from the start.
|
||||
- `24.06.2025` Switched to microseconds for tracking file changes during sync. Gap between consecutive files creation is more than enough - ranging from 5000μs to 1000μs. We didn't go for nanosec because js is having troubles with int64 precision. Added later. Linux is using cached kernel time, which is updated at `CONFIG_HZ` interval (`grep CONFIG_HZ /boot/config-$(uname -r)`), in my case the value is 1000 (1ms). Most real-world operations operations are spaced much further apart than 1ms due to: user interaction, network latency, disk i/o. We might only have issue if we update files inside an effective/native loop.
|
||||
- `16.06.2025` I believe it's time to make our knowledge base cross-platform, by forbidding characters like ":?<>*" in filenames. These characters aren't allowed in some environments (like Windows, PWA).
|
||||
- `14.06.2025` I wanted bot-like functionality in browser. I didn't want to re-write well-tested code in TypeScript, so I used wasm~~. And it worked perfectly good.
|
||||
- `12.06.2025` We use Telegram bot as distract-free write-only entrance to our knowledge base. The only issue is, it is not as wildly popular in EU/USA. I've come to the idea that we can transform app.files.md to a chat once we decrease the window size! Would be default behaviour on mobiles.
|
||||
- `04.06.2025` Introduced append-only log for syncing. Stateless sync is tricky to implement - we would have to send all files in every request. Since we're only renaming on server - we'll only track renames.
|
||||
- `04.06.2025` For content-only sync (no renames/deletes) we don't store any state on server, we compare hashes & last ctimes.
|
||||
- `11.11.2024` Removed Wikilinks support. Only plain Markdown links, our knowledge base must be interoperable.
|
||||
- `26.10.2024` Updates are now processed sequentially on per-user basis. Because there were some race conditions on concurrent file writings. Also we faced out-of-order forwarded messages processing, and it was impossible to collapse them to one message.
|
||||
- `06.10.2024` **Removed fyne.io**. At first, I wanted a lightweight alternative to Electron, and fyne.io seemed to be an ideal candidate. After a few days working with it 80% of bot functionality was implemented, and I was pretty happy with it. The thing is, to implement the rest of the functionality, we would have to apply A TREMENDOUS amount of effort. I am talking tiny details such as scrolling, emojis rendering, text selecting behaviour, links support, etc. And in future we would have to implement image uploading and markdown/html renderer, which would be also painful in such non-webview based toolkit. As much as I hate using the web stack for the desktop applications, it doesn't seem like we have a choice. Let's try wails.io.
|
||||
- `09.09.2024` We use vendoring for dependencies. We want all our few dependencies to be in the repo, so we don't care about blocked/removed dependencies. Our repository is the self-sufficient source of truth.
|
||||
- `01.09.2025` We use granular locks (in db, journal, userconfig) instead of one global per user lock so to avoid bottlenecks. Workers might use 3rd party API like ChatGPT, and we don't want to hold user's lock all that time. **PATCHED**, we added sequential per-user updates processing, `bot` can't cause RCs on its own, but `bot` & `worker` can, so we should continue using granular locks.
|
||||
- `20.08.2024` We read every userconfig value from the config file on every access. We don't need load/save whole config before/after `bot.Answer()` method. We have to reread it every time we need to change it, so we don't write back any stale data. Let's imagine we load config only once before `bot.Answer()`, next, we may have significant networking delays in `bot.Answer()` (let's say 2 seconds when making external requests), there are good changes that during those 2 seconds `worker.MoveDueTasks()` will modify `userconfig.Schedule`, causing data race (after bot's answer we write back stale data). And we don't want our schedule lost.
|
||||
- `08.08.2024` Sanitize Early, we gave up sanitizing in Path method. That's an unexpected behaviour - it breaks paths. We should sanitize everything as soon as we received. Most commands work with md5 hashes, for such cases no sanitize is needed.
|
||||
- `13.07.2024` `gofumpt` for stricter formatting. `gofumpt` is happy with a subset of the formats that gofmt is happy with. The less we have to choose between different formating options, the better.
|
||||
- `13.07.2024` FS's structure should have userFS name, to reflect the fact it user user-namespaced.
|
||||
- `09.07.2024` Note term is way too vague. Let's try to use "file" term, without any high level abstraction (like note).
|
||||
- `08.07.2024` Gave up on AST parsing/rendering. We had lots of corner cases via AST and the code was way complex. Markdown isn't that hard to parse, we can do it via good old straigforward code. We have 3x times less code now, and it is far less mentally taxing to understand. We did the same for MD->HTML conversion. Telegram doesn't support whole range of HTML tags, so it was easier to write our own md-to-html converter.
|
||||
- `08.07.2024` Adherence to Tolerant Reader principles. If enconunter gibberish during parsing - we skip it, but if we encounter flags of valid data (let's say `###`) but data itself is invalid - we panic. TODO preserve gibberish during read-write cycle.
|
||||
- `07.07.2024` Usage of https://github.com/rivo/uniseg. In Go, strings are read-only slices of bytes. They can be turned into Unicode code points using the for loop or by casting: []rune(str). However, multiple code points may be combined into one user-perceived character or what the Unicode specification calls "grapheme cluster". For example, white circle "⚪" has two runes, but one grapheme cluster.
|
||||
- `13.06.2024` Markdown to HTML conversion. User can have invalid Markdown in his notes, and TG API would fail to send invalid Markdown directly. So, first we escape HTML, then we convert user's Markdown to HTML and finally send it via Telegram API as HTML.
|
||||
- `13.06.2023` File hashing. Everywhere where we have user input - we should use fs.hash, otherwise we get long filenames, and tg returns `INVALID_DATA` error (callbackData max 64 bytes).
|
||||
- `13.06.2023` Introduced `db.go`. We had to abstract away Redis anyway (otherwise it's hard to write tests).
|
||||
- `13.03.2023` Package db.go doesn't store userID (we often use it separately...) Do we? Maybe we gonna use it without userID (like global bot stats?). Added: moved userID to class. Maybe in later we'll need this class outside of user's scope, but let's stay in the future. :)
|
||||
- `13.06.2023` We can't ucfist filename in fs.Put - what if that was user-created file (outside the bot), i.e. it comes with lowercase.
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目:`zakirullin/files.md`
|
||||
- 原始仓库:https://github.com/zakirullin/files.md
|
||||
- 导入方式:上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源,不代表 WeHub 是原项目作者
|
||||
+387
@@ -0,0 +1,387 @@
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/icon.png" alt="Files.md icon" title="Files.md" align="right" height="76" />
|
||||
|
||||
# Files.md
|
||||
一处私密、安静的思考空间。一款只为你 `.md` 文件而生的简单应用。
|
||||
|
||||
<p align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/appdark.png" alt="Files.md screenshot" title="Files.md"/>
|
||||
<a href="README.md">English</a> · 简体中文
|
||||
</p>
|
||||
|
||||
你的整段人生,都可以放在这里:
|
||||
- 📌 笔记
|
||||
- 📝 文档、项目
|
||||
- 💚 日记、习惯
|
||||
- ✅ 清单、任务
|
||||
|
||||
全部以纯 `.md` 文件保存,本地优先,对 LLM 友好。**隐私无虞——任何数据都不会上传到服务器**。
|
||||
|
||||
> 让你的数据,以纯粹的本地文件形式属于你。
|
||||
> 让打开这些文件的软件,也真正属于你。
|
||||
> 用文件和你自己的大脑,去培育知识。
|
||||
> 借助 LLM,去打磨文件周围的软件。
|
||||
> **属于自己的纯文件与软件,才能穿越时代。**
|
||||
|
||||
线上体验:[app.files.md](https://app.files.md)(Beta)。主站:[files.md](https://files.md)。
|
||||
|
||||
你可以把它用作第二大脑、卡片盒笔记(Zettelkasten)、笔记、日记、任务、清单等等。
|
||||
|
||||
**这个项目我已经做了 5 年。** 欢迎 **[在 GitHub 上支持作者 💚](https://github.com/sponsors/zakirullin)**。
|
||||
|
||||
[随手记下想法](#随手记下想法) · [如何深入思考](#如何深入思考) · [第二大脑?](#第二大脑) · [日记、任务、清单](#日记)
|
||||
|
||||
## 又一个笔记应用?
|
||||
也算是。但这一次:
|
||||
- 只保留必要功能,**约束孕育创造力**
|
||||
- 无需安装,一个浏览器就够了
|
||||
- 离线可用
|
||||
- **本地优先,文件不会离开你的设备**
|
||||
- 自由开源
|
||||
- 代码极其简洁。**一个人或一个 LLM 能把整个项目装进脑子里**
|
||||
- 换句话说,这个项目具有[低认知负荷](https://github.com/zakirullin/cognitive-load)
|
||||
- 代码库为 LLM 留好了扩展空间,随你按需添砖加瓦
|
||||
- 便携,没有任何构建系统,直接打开 `web/index.html` 即可
|
||||
- 同步开箱即用,但完全可选
|
||||
- 服务器就是一个二进制文件(也可以直接用 iCloud / Dropbox / Google Drive 同步)
|
||||
- Telegram 机器人,方便你在路上随时访问文件
|
||||
|
||||
## 如何使用
|
||||
- 打开 [app.files.md](https://app.files.md)(推荐 Chrome)
|
||||
- 点击地址栏右侧的 "Install files.md":
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/install.png" alt="Install" title="Install" width="350"/>
|
||||
</div>
|
||||
|
||||
- 打开一个本地文件夹,让改动持久化保存
|
||||
- 偶尔按 `Cmd+Shift+R` 强制刷新一下,拿到最新更新
|
||||
|
||||
## 如何同步
|
||||
| 方案 | 文件存放在哪儿 | 多设备同步 | 是否需要服务器 | 适合的人 |
|
||||
|-------------------------------------------------------|--------------------------------|---------------|--------------------------|------------------------------------------------|
|
||||
| **本地优先**,`app.files.md` 完全不发送任何数据 | 你设备上的某个文件夹 | 否 | 不需要 | 追求极致隐私,数据完全留在本地 |
|
||||
| **云盘文件夹同步**(iCloud / Dropbox / Google Drive) | 你现有的云盘文件夹 | 是 | 不需要(由云服务商负责) | 不想自己跑服务器,也想多设备同步 |
|
||||
| **[自托管同步服务器](docs/your-own-server.md)** | 你自己(或局域网内)的服务器 | 是 | 一个 Go 二进制 | 想在自家或公司网络里多设备同步 |
|
||||
| **托管同步服务器** | 我们托管的服务器 | 是 | `api.files.md` | 想立刻上手,不折腾任何配置 |
|
||||
|
||||
## 随手记下想法
|
||||
你可以打开聊天框,随手把脑子里冒出来的想法记下来。
|
||||
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/app.png" alt="Files.md screenshot" title="Files.md"/>
|
||||
|
||||
打开聊天框,发一条消息:
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/note1.png" width="350"/>
|
||||
</div>
|
||||
|
||||
挑一个去处(也可以稍后再决定):
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/note2.png" width="350"/>
|
||||
</div>
|
||||
|
||||
**这条流程足够让你顺手收下笔记、任务、日记和清单。**
|
||||
|
||||
## 在 Telegram 机器人里保存
|
||||
打开聊天,输入点东西,按 `Enter`:
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/bot.gif" alt="Saving things in bot" title="Saving things in bot" width="350"/>
|
||||
</div>
|
||||
|
||||
就这样,你的笔记已经保存到一个 `.md` 文件里了。
|
||||
|
||||
机器人的完整功能:
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/bot.png" width="700"/>
|
||||
</div>
|
||||
|
||||
别担心——默认情况下它比看上去简单得多。
|
||||
|
||||
[Telegram 机器人入口](https://t.me/FilesMDBot)。**其他即时通讯工具也会陆续支持。**
|
||||
|
||||
## 如何深入思考
|
||||
**让想法相连,让它们彼此叠加,并真正想透。**
|
||||
|
||||
1) 我用 [app.files.md](https://app.files.md) 来沉淀关于大脑和软件开发的认识
|
||||
2) 我把新笔记放进 `brain` 或 `dev` 文件夹,一条笔记只承载一个想法
|
||||
3) 在 Web 应用里,把彼此相关的笔记连接起来(按 `[` 即可)
|
||||
4) 所有东西最终都连成一片,正如我们的大脑
|
||||
5) **我会花时间在这些笔记之间穿行、反复琢磨**
|
||||
6) 久而久之,有些 `brain` 里的笔记和 `dev` 里的笔记开始看起来意外地相关
|
||||
7) 跨领域的这种连接,会激发出洞察
|
||||
8) 我据此写了一篇文章:[软件开发中的认知负荷](https://github.com/zakirullin/cognitive-load)
|
||||
|
||||
这个过程帮助我:
|
||||
- **深入思考**(在 AI 时代尤其重要)
|
||||
- **系统性地思考,看见更完整的图景**
|
||||
- **写出更有洞见的文字**
|
||||
|
||||
要做到以上几点,**你必须用自己的脑子去想**,而不是依赖什么花哨模板或者 AI 工作流。
|
||||
|
||||
- 一开始什么结构都别建,0 个文件夹
|
||||
- 一条笔记只承载一个想法
|
||||
- 每条笔记要在脱离上下文时也能被读懂
|
||||
- 学到新东西就立刻用上,别留给"未来的自己"
|
||||
- 把相关的笔记彼此链接起来
|
||||
- **常回去翻翻,再想想**
|
||||
|
||||
我和我的朋友用这套朴素的方法已经五年了,效果一直很好。
|
||||
|
||||
## 第二大脑?
|
||||
我想引用 [I Deleted My Second Brain](https://web.archive.org/web/20260518143828/https://www.joanwestenberg.com/i-deleted-my-second-brain-692aa40d59d5f06dd5131e43/) 里的一段话:
|
||||
|
||||
> Obsidian 是一款出色的软件,我深爱它。但任何东西失去节制都会变成陷阱。嵌套文件夹里的 Markdown 文件、追踪你生产力的插件、暗示某种全知感的关系图谱——看着自己的笔记编织成星座般的网络,会让人产生一种掌控的幻觉。但星座只是投影。它们讲故事,却并不等于理解。
|
||||
>
|
||||
> 刚开始使用知识管理工具时,我以为自己是在解决"遗忘"这件事。后来,我以为自己解决的是"整合"。
|
||||
>
|
||||
> **最终,我意识到我创造了一个新问题:拖延。系统越是壮大,我就越是把真正的思考工作推给某个未来的自己——那个会整理、贴标签、提炼、淘金的自己。**
|
||||
>
|
||||
> **而那个自己从未到来。**
|
||||
|
||||
第二大脑是一个迷人的概念。
|
||||
那些进阶模板、插件、AI 工作流……
|
||||
让人忍不住想把整个互联网的智慧都打包收下。
|
||||
这个整洁的系统看起来真美。每多一条笔记,多巴胺就涌出来一些。
|
||||
_第二大脑越来越好。_
|
||||
|
||||
可是,**第一大脑从来没有真正变得更聪明。**
|
||||
这才是问题——在 AI 时代,你的第一大脑前所未有地宝贵。
|
||||
|
||||
用**你的脑子**去把笔记想透。
|
||||
工具并不重要,思考才是。
|
||||
|
||||
新增一条笔记之前,先试着回答这两个问题:
|
||||
- 这则新知识能不能磨利我的判断、拓宽我的分类框架?
|
||||
- 拥有这份知识后,我能用不同的视角看世界吗?
|
||||
|
||||
## 笔记反而会阻碍体验
|
||||
- 一边读一边记,很容易让我们误以为自己理解了文本
|
||||
- 我们以为自己理解了,但其实**只是知道**
|
||||
- **而当"知道"变得足够多,我们甚至会觉得自己已经"做过了"(或者起码尝试过了)**
|
||||
|
||||
最糟糕的是,**我们因此不再让新的体验发生——因为我们"已经知道了"**。这就是一道知识的高墙。生活给了我们去亲身经历的机会,我们却拒绝,理由是"我已经知道了"。
|
||||
|
||||
## 通过阅读和做笔记自我疗愈?🧘
|
||||
**在情绪层面受到的伤害,必须在情绪层面治愈。**
|
||||
|
||||
不是靠脑力劳动和做笔记。
|
||||
没有行动的阅读,是娱乐,是一种拖延的形式。
|
||||
**再多自助类书籍也治不好情感的伤口。**
|
||||
真正有帮助的,是心理治疗、改写练习与椅子工作。是冥想。
|
||||
**疗愈来自感受。**
|
||||
|
||||
## 什么时候应该做笔记
|
||||
如果你的目标是:
|
||||
- 对某件事建立更深入、更有结构的理解
|
||||
- 做研究
|
||||
- 写一篇文章或一本书
|
||||
|
||||
那么记笔记完全合适。
|
||||
|
||||
## 日记
|
||||
对什么事情感觉良好?发一条消息。
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/journal.png" title="Journaling" width="350"/>
|
||||
</div>
|
||||
|
||||
然后点 "To Journal"。
|
||||
或者直接在消息末尾加上 ` jj` 或 ` жж`。
|
||||
|
||||
你的记录会被妥帖地存进 `journal/YYYY.MM Month.md`。
|
||||
|
||||
## 任务 ✅
|
||||
你正心流满满地工作着。
|
||||
同事突然让你发一份报告。
|
||||
把这件事一直挂在脑子里很耗能。
|
||||
丢一条消息,继续保持心流。
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/zakirullin/files.md/raw/main/web/img/task.png" title="Adding a task" width="350"/>
|
||||
</div>
|
||||
|
||||
只添加小而可执行的事项。
|
||||
不要写"规划一次假期"那种。
|
||||
写下"反正要做的事情"的第一小步即可。
|
||||
**写你愿意去做的,不是你只是想象会去做但还没动力的。**
|
||||
你的任务列表不该带来负罪感。
|
||||
|
||||
要稍后再做的事情,按 "To Later"。
|
||||
|
||||
## 清单 🛒
|
||||
朋友推荐了一本书给你。
|
||||
家里的黄油和面包没了。
|
||||
这些事一直挂在脑子里,会让人心累。
|
||||
|
||||
丢进聊天框,再转到对应的清单里。
|
||||
|
||||
## 文件结构
|
||||
你不必费心想结构,它是预定义的。
|
||||
当然,你也完全可以按自己的喜好来。
|
||||
|
||||
- 聊天:`Chat.md`
|
||||
- 笔记:`brain/Note.md`、`<category>/*.md`
|
||||
- 清单:`Read.md`、`Watch.md`、`Shop.md`、`MyChecklist_.md`
|
||||
- 日记:`journal/2024.08 August.md`
|
||||
- 任务:`Later.md`
|
||||
- 习惯:`habits/Ate consciously.md`、`habits/*.md`
|
||||
- 图片:`media/*`(png、jpg、webp、gif)
|
||||
- 归档:`archive/*.md`
|
||||
- 配置:`config.json`
|
||||
|
||||
这份结构也可以在 [files.md/AGENTS.md](https://files.md/AGENTS.md) 取到。
|
||||
把它作为 `AGENTS.md` 放进你的知识库——你的文件就成了 AI 可以参与的鲜活个人空间。
|
||||
|
||||
## 快捷键
|
||||
|
||||
| 快捷键 | 操作 |
|
||||
|----------------------------------------|-------------------------------|
|
||||
| `[` | 插入到某个文件的链接 |
|
||||
| `Cmd+K` / `Ctrl+K` | 打开文件搜索框 |
|
||||
| `Cmd+N` / `Ctrl+N` | 新建文件 |
|
||||
| `Cmd+M` / `Ctrl+M` | 移动文件 |
|
||||
| `Cmd+D` / `Ctrl+D` | 删除文件 |
|
||||
| `Cmd+Enter` / `Ctrl+Enter` | 打开聊天 |
|
||||
| `Cmd+Shift+Enter` / `Ctrl+Shift+Enter` | 切换聊天弹窗 |
|
||||
| `Cmd+[` / `Ctrl+[` | 跳到上一个文件 |
|
||||
| `Cmd+]` / `Ctrl+]` | 跳到下一个文件 |
|
||||
| `Cmd+~` / `Ctrl+~` | 切换侧边栏 |
|
||||
| `Cmd+B` / `Ctrl+B` | 切换**粗体** |
|
||||
| `Cmd+I` / `Ctrl+I` | 切换*斜体* |
|
||||
| `Cmd+Y` / `Ctrl+Y` | 插入复选框 |
|
||||
| `Cmd/Ctrl` + `Click` | 复制纯文本 / 打开链接 |
|
||||
| `Ctrl+Cmd+Space` | 插入 emoji(macOS) |
|
||||
|
||||
## 一些好用的脚本
|
||||
所有脚本都在 `cmd` 目录下,**请在你的笔记文件夹里运行**。请先安装 [Go](https://go.dev/doc/install)。
|
||||
|
||||
### 把 Whoop 数据写进日记
|
||||
```
|
||||
go run /abs/path/to/files.md/cmd/whoop/whoop.go
|
||||
```
|
||||
|
||||
### 把 wikilinks 转成 Markdown 链接
|
||||
把 `[[wikilinks]]` 转成标准的 `[Name](/path.md)`(支持 `--dry-run`):
|
||||
```
|
||||
go run /abs/path/to/files.md/cmd/tomdlinks/tomdlinks.go .
|
||||
```
|
||||
|
||||
### 插入反向链接
|
||||
在被引用的文件里加上回链(支持 `--dry-run`):
|
||||
```
|
||||
go run /abs/path/to/files.md/cmd/backlink/backlink.go
|
||||
```
|
||||
|
||||
### 调整日记时间戳
|
||||
把日记文件中的时间戳整体平移 N 小时(换时区后很实用):
|
||||
```
|
||||
go run /abs/path/to/files.md/cmd/shifttime/shifttime.go
|
||||
```
|
||||
|
||||
## 文档
|
||||
[在自己的服务器上部署](docs/your-own-server.md)
|
||||
[Telegram 机器人](docs/bot.md)
|
||||
[同步流程](docs/sync-flow.md)
|
||||
[集成测试](docs/integration-tests.md)
|
||||
|
||||
## 仓库结构
|
||||
- `web` —— Web 应用(PWA),入口是 `index.html`
|
||||
- `web/lib` —— 前端依赖库
|
||||
- `cmd/server` —— 服务端入口
|
||||
- `cmd/*/` —— 处理 `.md` 文件的脚本
|
||||
- `server/bot.go` —— 机器人
|
||||
- `server/sync/` —— 同步 API 服务端
|
||||
- `vendor` —— 后端依赖
|
||||
- `tests` —— 端到端测试,会同时覆盖 Web 应用和服务端
|
||||
|
||||
## 如何贡献
|
||||
- 初级开发者也应能读懂代码
|
||||
- 理想情况下,每个 PR 都应该删减或简化代码,而不是新增
|
||||
- **代码越少,我们就越灵活**
|
||||
- 所有依赖都是我们自己的代码与责任,能不引就不引
|
||||
- 代码应当自洽,所以 `vendor` 和 `web/lib` 都在仓库里
|
||||
- **我们真的需要这个功能吗?它能帮我们做"真正的事",还是只为了一时的多巴胺?**
|
||||
|
||||
更完整的原则请参考 [这份指南](https://github.com/zakirullin/cognitive-load)。
|
||||
|
||||
## 后端规范
|
||||
- 我们写**测试**
|
||||
- 我们不用 `get*` 前缀命名方法
|
||||
- 不要 panic,错误本身就是业务逻辑的一部分
|
||||
- 如果要忽略某个 error,必须留下解释 WHY 的注释
|
||||
- 我们一律包装 error,并补上方法层面的上下文
|
||||
- 客户端代码里不使用迭代器
|
||||
- 优先使用真实实现或至少 fake,而不是 mock / stub
|
||||
- import 只在与其他 import 冲突时才重命名
|
||||
- **为了可移植性,所有数据都存在纯 `.md` 文件里**
|
||||
|
||||
## 前端规范
|
||||
- 如果你就地修改了第三方库,请用 `PATCHED` 关键字标注
|
||||
- **要是有一天我们能用自己的小实现替换掉 `CodeMirror`,那就太棒了**
|
||||
- 没有构建系统,**十年后我们打开 `/web/index.html`,它仍然要能直接跑**
|
||||
- 别忘了:在"检查锁"和"获取锁"之间出现的 `await` 会引入竞态
|
||||
- 别写脆弱的 e2e 测试。先是带来负面情绪,然后我们就再也不跑测试了
|
||||
- 大多数 bug 都源于异步流程中途被打断时的竞态条件
|
||||
|
||||
## 术语表
|
||||
- `filename` —— 带扩展名的文件名,例如 "note.md"(**用它作为 ID**)
|
||||
- `header` —— 去掉扩展名并首字母大写后的文件名,例如 "Note"
|
||||
- `body` —— 文件内容
|
||||
- `dir` —— 用来按类别存放笔记的目录,例如 "happiness"
|
||||
- `userID` —— 即 chatID。大多数场景下我们就直接用 chatID(私聊机器人)
|
||||
- 文件的 `ctime` —— 数据块或元数据变更时间:所有权、位置、文件类型、权限变更都会触发。父目录改名不会触发,移动文件会触发,重命名会触发。我们需要它来追踪文件的位置变化,比如什么时候被移到归档、追踪任务的"愤怒度"等
|
||||
- 文件的 `mtime` —— 文件内容最近一次被修改的时间。和 `ctime` 不同,所有权、权限、改名都不会影响它。我们用它来做同步
|
||||
- 目录的 `ctime` —— 增删文件或子目录会触发(类似 `mtime`,再加上 inode 的变化,比如改名)
|
||||
|
||||
每个文件都可以通过文件名与所在目录唯一定位。我们只支持一层嵌套。
|
||||
|
||||
## 性能
|
||||
项目跑起来飞快 :) 如果你怕用文件或互斥锁会带来性能开销,看一眼这张对比:
|
||||
```
|
||||
Mutex 加 / 解锁 = 25 ns
|
||||
从 SSD 随机读 4K = 150,000 ns
|
||||
1 ms = 1,000,000 ns
|
||||
```
|
||||
|
||||
## ADR(架构决策记录)
|
||||
- `06.05.2026` 从 Today.md 切回 Chat.md。用户访谈发现"chat"这个概念用户更容易理解。而且 "open chat" 这句话在机器人和网页应用里语义都一致。
|
||||
- `02.05.2026` hide-token 现在在每次变更时同步运行;之前的 100ms 防抖会让链接和格式化文本在逐字删除时出现抖动。
|
||||
- `06.05.2026` 把 Inbox.md 和 Today.md 合并成了 Today.md。"Inbox"这个名字太抽象、太"生产力"和 GTD 味道。我想要的是从容和简洁。Today 更像"我活在聊天里的那一页"。
|
||||
- `23.04.2026` 把 API_HOST、APP_HOST 改成了 API_URL、APP_URL。不同环境下,配置里能携带 schema 等更多信息总归更好。
|
||||
- `22.04.2026` 机器人里的 Inbox 条目改用稳定的内容哈希识别(对去掉 `- [ ] ` / `- [x] ` 标记后的块取 `fs.Hash`),而不是位置序号。这样即便中间有条目被新增/删除/完成,按钮也仍指向正确的那一行。
|
||||
- `06.05.2026` 同时存在 "to inbox" 和 "to chat" 两个按钮,心智上挺累。也很难轻松地把任务往 chat 里随手一丢——因为它会先落到 inbox,还要多点一次。这一次多余的点击,正是让"添加新任务"变得令人厌烦的原因。我放弃了两条不同的流程,现在一切都先进 inbox,每一项都是 Markdown 复选框。额外的好处:PWA 默认展示 chat 里的任务,特别顺手。也许 "inbox" 这个词心智负担太重,"chat" 听起来更轻松。再观察一下。
|
||||
- `11.04.2026` 虽然我希望以纯 Markdown 链接形式存储链接,但视觉上我希望它们能像极简的 `[link]` 一样工作。所以当光标停在该行时,我会隐藏 `(...)` 那部分。仅对指向 md 文件的链接生效。
|
||||
- `11.04.2026` 又把标准 Markdown 链接拉回来了。我希望这个知识库跨平台可用,它必须在 GitHub 上也能正常显示。
|
||||
- `05.04.2026` 试过把 `web/*` 移到根目录以求简洁,结果发现是个坏主意——必须有一个显式的目录,作为服务器上的公开 DOCROOT。
|
||||
- `19.04.2026` 改成了 `[link]` 形式的链接。`[link](full%20path)` 太重也太突兀,我们也不想处理路径变更。
|
||||
- `21.09.2025` 移除了 WASM。我当时遇到一个 bug:从 Inbox.txt 删除一条消息后,按"move to file"没有把它写到目标文件里。我没复现出来,但发现了一大堆复杂度:JS -> Go(writeFile)-> Go 等待来自 JS 的 promise -> 中间的 JS Golang 运行时 -> JS(writeFile)-> Go(promise 返回)-> 把结果发回 JS。而且这一切还得跑在独立 goroutine 里,因为 WASM 和 JS 共用一个线程。再加上 Go 的 WASM 仍是实验性的。组件太多、不确定性太大。当时我不愿意用 JS 重写已经能用的逻辑,于是用了 WASM 撑了一段时间。现在该用 JS 把它重新实现,把这一整摞复杂度都甩掉。另外 `inbox.wasm` 大约 8MB,我希望整个应用尽可能小。
|
||||
- `11.07.2025` 决定先用 OPFS 作为文件系统的初始驱动。浏览器兼容性更好,对用户更省心。应用默认以 OPFS 启动;用户需要时可以打开一个本地目录,切到 Local FileSystem API。DirHandle 会被保存到 IndexedDB,下次复用。
|
||||
- `08.07.2025` 根目录现在是 `/` 而不是空字符串。Web 应用里所有文件按路径唯一标识,不再用 `dir + filename`,并限制为一层嵌套。
|
||||
- `11.04.2026` Dropbox 会改动新创建文件的某些元数据,因此 `ctime` 会被改写。我考虑过把同步切到 `mtime`,但这样就检测不出重命名(虽然我们另有机制检测重命名),而 `mtime` 整体上更可靠:权限/所有权变更也不会触发同步。最终切换到了 `mtime`。`mtime` 用于内容级同步,`ctime` 用于追加式同步日志(重命名 / 删除)。另外 `mtime` 可以从 `.git/archive` 里恢复,`ctime` 不行。
|
||||
- `30.06.2025` 决定把所有流程,包括待办列表,都迁移到 Chat.md。补充:这种流程不太方便处理多行任务,可能需要同时支持文件和索引。两种实现思路:参数统一编码,复用同一个命令处理器加 if 分支;或者用不同的命令处理器分别处理 chat / 文件。我选了后者。补充:如果走分流,"move to" 按钮的配置会更复杂。补充:也许在"move to file"时把文件移回 Chat.md,复用现有流程?补充:目前看来还不错。我们的 chat.md 充当一个仅追加日志。额外的好处是:即便某些流程(比如安排时间 / 移动)没走完,内容也保留在日志里,下次可以继续。
|
||||
- `06.05.2026` 所有新进消息现在默认进 Chat.md。在此之前它们会被移到 `/chat` 并变成任务,这对简单的 to-do 列表很合适,但对其他用例不够灵活。我意识到开会时,我真正需要的就是一个简单的输入框,把脑子里乱七八糟的东西先丢进去,不必立即决定怎么处理,回头再整理就好。它们可能是任务、可能是日记、也可能就是文件。同时保持一条简单到无需理解的默认流程,对用户更友好——所有消息都丢进同一个文件,仅此而已。
|
||||
- `27.06.2025` Chat 现在默认是"一个大文件"模式,也就是把所有消息都堆进同一个文件,不做更多事。还是那句话:从最简单的流程开始,不要一上来就把用户淹没。补充:如果选了完整模式,我们就得提前建好目录,"to habits"、"to read/shop" 之类才能工作。用户用不上的话,把目录删了我们就不再自动重建(不像"按需创建"模式那样)。所以"按需创建"并不适合所有场景。
|
||||
- `26.06.2025` 之前是提前把所有目录都建好,现在改成按需创建。这样不会一上来就把用户的知识库塞得乱七八糟。
|
||||
- `24.06.2025` 同步时改用微秒级别追踪文件变化。连续创建文件之间的间隔通常在 1000μs–5000μs,足够区分。没用纳秒是因为 JS 处理 int64 精度有问题。补充:Linux 用的是内核缓存时间,按 `CONFIG_HZ` 间隔更新(`grep CONFIG_HZ /boot/config-$(uname -r)`),我这里是 1000(1ms)。实际操作之间的间隔通常远大于 1ms:用户交互、网络延迟、磁盘 I/O 都会拉长间隔。只有在原生紧凑循环里更新文件才可能踩坑。
|
||||
- `16.06.2025` 我觉得是时候让知识库真正跨平台了——禁掉文件名里的 `:?<>*` 等字符。这些字符在某些环境(Windows、PWA)里本来就不允许。
|
||||
- `15.09.2025` 我想在浏览器里有类机器人的能力。又不想把已经被良好测试过的代码用 TypeScript 重写一遍,所以用了 wasm。它确实跑得挺好。
|
||||
- `12.06.2025` 我们把 Telegram 机器人当作通往知识库的"无干扰、只写"入口。唯一的问题是它在欧美没那么流行。后来想到:把窗口缩小后,app.files.md 也可以直接呈现成一个聊天界面!这将是移动端的默认形态。
|
||||
- `04.06.2025` 引入了用于同步的追加式日志。无状态同步实现起来很费劲——每次请求都要把所有文件发过去。既然服务器端只处理重命名,那我们只追踪重命名就好。
|
||||
- `04.06.2025` 仅做内容同步(不含重命名 / 删除)时,服务端不存任何状态,只比较哈希和最近的 ctime。
|
||||
- `11.11.2024` 移除了 Wikilinks 支持。只保留纯 Markdown 链接,我们的知识库必须保持互操作性。
|
||||
- `26.10.2024` 更新现在按用户串行处理。原因是并发写文件出现过竞态。同时还遇到过转发消息乱序的问题,导致无法合并成一条。
|
||||
- `06.10.2024` **移除了 fyne.io**。一开始我只是想找一个比 Electron 更轻量的方案,fyne.io 看起来是个理想候选。用了几天,机器人 80% 的功能已经搬过来了,体验也不错。但要把剩下 20% 落地,得花上巨大的工夫——滚动行为、emoji 渲染、文本选择、链接支持等等细枝末节。未来还要做图片上传、Markdown/HTML 渲染,在一个不基于 webview 的工具集里这些会非常痛苦。尽管我个人挺反感用 Web 技术做桌面应用,但似乎没得选。试试 wails.io。
|
||||
- `09.09.2024` 我们用 vendoring 管理依赖。我希望所有的少量依赖都在仓库里,这样就不必担心依赖被屏蔽或下线。我们的仓库就是自给自足的"唯一可信源"。
|
||||
- `09.06.2025` 我们在 db、journal、userconfig 里使用细粒度锁,而不是给每个用户上一把全局锁,以避免瓶颈。Workers 可能要调外部 API(如 ChatGPT),不希望长期持有用户锁。**已 PATCHED**——加入了按用户串行更新,`bot` 自身不会导致竞态,但 `bot` 和 `worker` 之间仍可能竞态,所以继续保留细粒度锁。
|
||||
- `20.08.2024` 我们在每次访问时都从配置文件读取 userconfig 的每个字段。不需要在 `bot.Answer()` 前后整体加载/保存配置。修改时必须重新读取,避免写回过时数据。设想一下:如果只在 `bot.Answer()` 之前加载一次,而 `bot.Answer()` 中又有可观的网络延迟(比如对外部接口发请求要 2 秒),那这 2 秒内 `worker.MoveDueTasks()` 完全可能改写 `userconfig.Schedule`,引发数据竞争(机器人答完后写回的就成了过期数据),调度信息会丢失。
|
||||
- `08.08.2024` 早期清洗(Sanitize Early)。我们放弃了在 `Path` 方法里做清洗——那种行为非常违反直觉,会把路径搞坏。东西一收进来就应该清洗。大多数命令都基于 md5 哈希工作,那种场景不需要清洗。
|
||||
- `13.07.2024` 用 `gofumpt` 做更严格的格式化。`gofumpt` 接受的格式是 gofmt 的子集。在格式选项上越少做选择,越好。
|
||||
- `13.07.2024` FS 的结构里应该带上 `userFS` 名字,明确它是按用户命名空间隔离的。
|
||||
- `09.07.2024` "Note" 这个词太空泛。我们改用 "file",不再套用"笔记"这种高层抽象。
|
||||
- `13.08.2024` 放弃了 AST 解析 / 渲染。走 AST 时遇到太多边界情况,代码也愈发复杂。Markdown 并不那么难解析,老老实实写直白的代码就好。最终代码量降到原来的 1/3,理解起来也轻松多了。MD->HTML 转换我们做了同样的事。Telegram 并不支持完整的 HTML 标签,自己写一个 md-to-html 转换器反而更容易。
|
||||
- `08.07.2024` 遵循 Tolerant Reader 原则。解析时遇到乱码就跳过;但如果看到看起来合法的标记(比如 `###`)后面跟着无效数据,就直接 panic。TODO:在读写循环中保留这些乱码。
|
||||
- `07.07.2024` 引入了 https://github.com/rivo/uniseg。在 Go 里,string 是只读的字节切片,可以通过 `for` 或 `[]rune(str)` 转成 Unicode 码点。但多个码点可能被组合成一个"用户感知"的字符,也就是 Unicode 规范里说的"字素簇"(grapheme cluster)。例如白色圆圈 "⚪" 由两个 rune 组成,却是一个字素簇。
|
||||
- `07.09.2024` Markdown 转 HTML。用户笔记里可能有非法 Markdown,直接发给 TG API 会失败。所以我们先转义 HTML,再把用户的 Markdown 转成 HTML,最后通过 Telegram API 以 HTML 形式发出。
|
||||
- `13.06.2023` 文件名哈希化。所有出现用户输入的地方都应该用 `fs.hash`,否则文件名会变得很长,而 Telegram 会返回 `INVALID_DATA`(callbackData 最大 64 字节)。
|
||||
- `13.06.2023` 引入了 `db.go`。本来就需要把 Redis 抽象出来(不然测试不好写)。
|
||||
- `13.03.2023` `db.go` 这个包里不存 `userID`(我们经常单独使用 `userID`)……真的吗?也许哪天我们会在与用户无关的场景下用它(比如全局 bot 统计)。补充:之后把 `userID` 挪进了类里。未来如果在用户作用域之外还要复用这个类再说,眼下先这样。
|
||||
- `13.06.2023` 我们不能在 `fs.Put` 里给文件名做首字母大写——万一这是用户在外部自己建的文件(小写)就被改坏了。
|
||||
@@ -0,0 +1,145 @@
|
||||
// Package scripts Backlink is a small script which is meant to insert backlinks into our notes
|
||||
// You can run it manually on your knowledge base, or you can run it periodically
|
||||
// Should be run with working directory set to your root knowledge base
|
||||
// WARNING! Cases with "|" in urls aren't handled yet, so duplicate urls possible
|
||||
// We don't support relative paths, missing folder means root
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"golang.org/x/exp/slices"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dryRun := false
|
||||
for _, arg := range os.Args[1:] {
|
||||
if arg == "--dry-run" {
|
||||
dryRun = true
|
||||
}
|
||||
}
|
||||
|
||||
// [dir][note] => links referring to our note (backlinks)
|
||||
backlinks := make(map[string]map[string][]string)
|
||||
|
||||
userFS, err := fs.NewFS(".", afero.NewOsFs())
|
||||
if err != nil {
|
||||
fmt.Printf("Can't create FS: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
files, err := userFS.FilesAndDirs("")
|
||||
if err != nil {
|
||||
fmt.Printf("Can't get files and dirs: %s", err)
|
||||
return
|
||||
}
|
||||
dirs := fs.OnlyNoteDirs(fs.OnlyDirs(files))
|
||||
|
||||
// Run through files, if our file has link to some note, we add our current note's path to referred note's backlinks
|
||||
for _, dir := range dirs {
|
||||
notes, err := userFS.FilesAndDirs(dir.Name)
|
||||
if err != nil {
|
||||
fmt.Printf("Can't get notes: %s", err)
|
||||
}
|
||||
|
||||
notes = fs.OnlyUserMDFiles(notes)
|
||||
for _, note := range notes {
|
||||
if filepath.Ext(note.Name) != fs.MDExt {
|
||||
continue
|
||||
}
|
||||
|
||||
content, err := userFS.Read(dir.Name, note.Name)
|
||||
if err != nil {
|
||||
fmt.Printf("Can't get content: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
links := regexp.MustCompile(`\[\[(.+?)\]\]`)
|
||||
matches := links.FindAllStringSubmatch(content, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
link := match[1]
|
||||
if strings.Contains(link, "img/") {
|
||||
continue
|
||||
}
|
||||
// There are issues with "й" letter. Probably it has non-canonical encoding in mac FS
|
||||
link = string(norm.NFC.Bytes([]byte(link)))
|
||||
// We don't support labels
|
||||
link = strings.Split(link, "|")[0]
|
||||
|
||||
// dir/note
|
||||
parts := strings.Split(link, "/")
|
||||
|
||||
isInRootDir := len(parts) == 1
|
||||
if isInRootDir {
|
||||
// TODO implement support for root files
|
||||
continue
|
||||
}
|
||||
|
||||
refToDir := parts[0]
|
||||
refToNote := parts[1]
|
||||
|
||||
filename := note.Name
|
||||
filename = string(norm.NFC.Bytes([]byte(filename)))
|
||||
filename = strings.TrimSuffix(filename, fs.MDExt)
|
||||
link = fmt.Sprintf("%s/%s", dir.Name, filename)
|
||||
|
||||
if _, ok := backlinks[refToDir]; !ok {
|
||||
backlinks[refToDir] = make(map[string][]string)
|
||||
}
|
||||
backlinks[refToDir][refToNote] = append(backlinks[refToDir][refToNote], link)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for dir, notes := range backlinks {
|
||||
for note, links := range notes {
|
||||
for _, link := range links {
|
||||
content, err := userFS.Read(dir, note+fs.MDExt)
|
||||
if err != nil {
|
||||
fmt.Printf("Can't get target note '%s/%s.md':%s, backlinks: %v", dir, note, err, links)
|
||||
continue
|
||||
}
|
||||
|
||||
var existingLinks []string
|
||||
existingLinksRx := regexp.MustCompile(`\[\[(.*)\]\]`)
|
||||
matches := existingLinksRx.FindAllStringSubmatch(content, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) < 2 {
|
||||
continue
|
||||
}
|
||||
existingLink := strings.Split(match[1], "|")[0]
|
||||
existingLinks = append(existingLinks, existingLink)
|
||||
}
|
||||
|
||||
if slices.Contains(existingLinks, link) {
|
||||
continue
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
fmt.Printf("would add '%s' to '%s/%s'\n", link, dir, note)
|
||||
continue
|
||||
}
|
||||
|
||||
err = userFS.Write(dir, note+".md", fmt.Sprintf("%s\n[[%s]]", strings.TrimSpace(content), link))
|
||||
if err != nil {
|
||||
fmt.Printf("Can't put to file: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("ADD '%s' TO '%s/%s'\n", link, dir, note)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
// Package scripts Backlink is a small script which is meant to insert backlinks into our notes
|
||||
// You can run it manually on your knowledge base, or you can run it periodically
|
||||
// Should be run with working directory set to your root knowledge base
|
||||
// WARNING! Cases with "|" in urls aren't handled yet, so duplicate urls possible
|
||||
// We don't support relative paths, missing folder means root
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"golang.org/x/exp/slices"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// [dir][note] => links referring to our note (backlinks)
|
||||
backlinks := make(map[string]map[string][]string)
|
||||
|
||||
userFS, err := fs.NewFS(".", afero.NewOsFs())
|
||||
if err != nil {
|
||||
fmt.Printf("Can't create FS: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
files, err := userFS.FilesAndDirs("")
|
||||
if err != nil {
|
||||
fmt.Printf("Can't get files and dirs: %s", err)
|
||||
return
|
||||
}
|
||||
dirs := fs.OnlyNoteDirs(fs.OnlyDirs(files))
|
||||
|
||||
// Run through files, if our file has link to some note, we add our current note's path to referred note's backlinks
|
||||
for _, dir := range dirs {
|
||||
notes, err := userFS.FilesAndDirs(dir.Name)
|
||||
if err != nil {
|
||||
fmt.Printf("Can't get notes: %s", err)
|
||||
}
|
||||
|
||||
notes = fs.OnlyUserMDFiles(notes)
|
||||
for _, note := range notes {
|
||||
if filepath.Ext(note.Name) != fs.MDExt {
|
||||
continue
|
||||
}
|
||||
|
||||
content, err := userFS.Read(dir.Name, note.Name)
|
||||
if err != nil {
|
||||
fmt.Printf("Can't get content: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
links := regexp.MustCompile(`\[\[(.+?)\]\]`)
|
||||
matches := links.FindAllStringSubmatch(content, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
link := match[1]
|
||||
if strings.Contains(link, "img/") {
|
||||
continue
|
||||
}
|
||||
// There are issues with "й" letter. Probably it has non-canonical encoding in mac FS
|
||||
link = string(norm.NFC.Bytes([]byte(link)))
|
||||
// We don't support labels
|
||||
link = strings.Split(link, "|")[0]
|
||||
|
||||
// dir/note
|
||||
parts := strings.Split(link, "/")
|
||||
|
||||
isInRootDir := len(parts) == 1
|
||||
if isInRootDir {
|
||||
// TODO implement support for root files
|
||||
continue
|
||||
}
|
||||
|
||||
refToDir := parts[0]
|
||||
refToNote := parts[1]
|
||||
|
||||
filename := note.Name
|
||||
filename = string(norm.NFC.Bytes([]byte(filename)))
|
||||
filename = strings.TrimSuffix(filename, fs.MDExt)
|
||||
link = fmt.Sprintf("%s/%s", dir.Name, filename)
|
||||
|
||||
if _, ok := backlinks[refToDir]; !ok {
|
||||
backlinks[refToDir] = make(map[string][]string)
|
||||
}
|
||||
backlinks[refToDir][refToNote] = append(backlinks[refToDir][refToNote], link)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for dir, notes := range backlinks {
|
||||
for note, links := range notes {
|
||||
for _, link := range links {
|
||||
content, err := userFS.Read(dir, note+fs.MDExt)
|
||||
if err != nil {
|
||||
fmt.Printf("Can't get target note '%s/%s.md':%s, backlinks: %v", dir, note, err, links)
|
||||
continue
|
||||
}
|
||||
|
||||
var existingLinks []string
|
||||
existingLinksRx := regexp.MustCompile(`\[\[(.*)\]\]`)
|
||||
matches := existingLinksRx.FindAllStringSubmatch(content, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) < 2 {
|
||||
continue
|
||||
}
|
||||
existingLink := strings.Split(match[1], "|")[0]
|
||||
existingLinks = append(existingLinks, existingLink)
|
||||
}
|
||||
|
||||
if slices.Contains(existingLinks, link) {
|
||||
continue
|
||||
}
|
||||
|
||||
err = userFS.Write(dir, note+".md", fmt.Sprintf("%s\n[[%s]]", strings.TrimSpace(content), link))
|
||||
if err != nil {
|
||||
fmt.Printf("Can't put to file: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("ADD '%s' TO '%s/%s'\n", link, dir, note)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/spf13/afero"
|
||||
|
||||
"github.com/zakirullin/files.md/server"
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/db"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
"github.com/zakirullin/files.md/server/pkg/tg"
|
||||
"github.com/zakirullin/files.md/server/sync"
|
||||
"github.com/zakirullin/files.md/server/userconfig"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := godotenv.Load(); err != nil {
|
||||
fmt.Println("No .env file found, relying on process environment")
|
||||
}
|
||||
if err := config.LoadBotConfig(); err != nil {
|
||||
panic(fmt.Sprintf("Error loading cfg: %s\n", err))
|
||||
}
|
||||
// Save all renames and deletes to an append-only log.
|
||||
fs.LogRename = sync.LogRename
|
||||
fs.LogDelete = sync.LogDelete
|
||||
|
||||
// Launch the HTTP server, it does two things:
|
||||
// - Serves the PWA app
|
||||
// - Provides the API for synchronization (if API_URL is not empty)
|
||||
go sync.Serve(
|
||||
config.ServerCfg.APIHost(),
|
||||
config.ServerCfg.AppHost(),
|
||||
config.ServerCfg.ServerCertDir,
|
||||
config.ServerCfg.ServerLogFile,
|
||||
)
|
||||
|
||||
// Telegram bot is optional - server can run as web-only.
|
||||
api, err := tgbotapi.NewBotAPI(config.ServerCfg.BotAPIToken)
|
||||
if err != nil {
|
||||
fmt.Println("No Telegram bot token found, running web and api server only")
|
||||
select {} // block forever
|
||||
}
|
||||
telegram := tg.NewTG(api)
|
||||
|
||||
// If today or inbox was changed in web app, we need to send the updated items to the bot.
|
||||
sync.OnChatUpdate = func(userID int64) { updateBotHome(telegram, userID) }
|
||||
|
||||
// Due tasks scheduler
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
quit := make(chan struct{})
|
||||
defer func(quit chan struct{}) {
|
||||
close(quit)
|
||||
}(quit)
|
||||
go func(tg *tg.TG) {
|
||||
fsBackend := afero.NewOsFs()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
err := server.MoveDueTasks(config.ServerCfg.StorageDir, config.ServerCfg.ConfigFilename, fsBackend, telegram)
|
||||
if err != nil {
|
||||
fmt.Printf("Worker's error: %s\n", err)
|
||||
}
|
||||
|
||||
err = server.RemoveCompletedChecklistItems(config.ServerCfg.StorageDir, config.ServerCfg.ConfigFilename, fsBackend)
|
||||
if err != nil {
|
||||
fmt.Printf("Worker's error: %s\n", err)
|
||||
}
|
||||
case <-quit:
|
||||
ticker.Stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}(telegram)
|
||||
|
||||
infolog := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
|
||||
// Main bot loop.
|
||||
// Loop through updates from Telegram and process them sequentially in separate per-user goroutine.
|
||||
userChannels := make(map[int64]chan tgbotapi.Update)
|
||||
tgConfig := tgbotapi.NewUpdate(0)
|
||||
tgConfig.Timeout = 60 // TODO release, check if it's enough
|
||||
updates := api.GetUpdatesChan(tgConfig)
|
||||
for update := range updates {
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
slog.Error("Bot panic", "update", update, "err", r, "stacktrace", string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
|
||||
updJSON, _ := json.Marshal(update)
|
||||
infolog.Info("Bot update", "update", string(updJSON))
|
||||
|
||||
var userID int64
|
||||
upd := tg.NewTGUpd(update)
|
||||
channelID, channelIDExists := upd.ChannelID()
|
||||
if channelIDExists {
|
||||
userID, err = telegram.ChannelCreatorID(channelID)
|
||||
if err != nil {
|
||||
slog.Error("Bot error: can't get channel creator ID", "upd", string(updJSON), "err", err)
|
||||
}
|
||||
} else {
|
||||
userID = upd.UserID()
|
||||
}
|
||||
|
||||
userCh, channelIDExists := userChannels[userID]
|
||||
if !channelIDExists {
|
||||
userCh = make(chan tgbotapi.Update, 100)
|
||||
userChannels[userID] = userCh
|
||||
go supervisor(userID, userCh, telegram)
|
||||
}
|
||||
|
||||
userCh <- update
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// Runs per-user worker that listens for updates.
|
||||
// Restarts infinitely upon panics.
|
||||
func supervisor(userID int64, updates <-chan tgbotapi.Update, telegram *tg.TG) {
|
||||
for {
|
||||
func() {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
slog.Error("Bot panic", "userID", userID, "err", err, "stacktrace", string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
processUserUpdates(userID, updates, telegram)
|
||||
}()
|
||||
time.Sleep(time.Second)
|
||||
slog.Info("Restarting worker", "userID", userID)
|
||||
}
|
||||
}
|
||||
|
||||
func processUserUpdates(userID int64, updates <-chan tgbotapi.Update, telegram *tg.TG) {
|
||||
for update := range updates {
|
||||
upd := tg.NewTGUpd(update)
|
||||
|
||||
bot, err := newBot(telegram, userID)
|
||||
if err != nil {
|
||||
slog.Error("Bot error: can't create bot", "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := bot.Reply(upd); err != nil {
|
||||
if errors.Is(err, tg.ErrFileTooBig) {
|
||||
if _, sendErr := telegram.Send(userID, "This file is too big. Telegram allows bots to download files up to 20MB.", nil, ""); sendErr != nil {
|
||||
slog.Error("Bot error: can't send file-too-big notice", "err", sendErr)
|
||||
}
|
||||
}
|
||||
slog.Error("Bot error", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newBot(telegram *tg.TG, userID int64) (*server.Bot, error) {
|
||||
userFS, err := fs.NewUserFS(userID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't create fs: %w", err)
|
||||
}
|
||||
err = userFS.CreateSystemDirs()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't create user dirs: %w", err)
|
||||
}
|
||||
|
||||
confFilename := config.ServerCfg.ConfigFilename
|
||||
userconf := userconfig.NewConfig(userFS, userID, confFilename)
|
||||
err = userconf.CreateDefaultIfNotExists()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't create default user config: %w", err)
|
||||
}
|
||||
|
||||
bot := server.NewBot(userID, telegram, userFS, db.NewDB(userID), userconf)
|
||||
|
||||
return bot, nil
|
||||
}
|
||||
|
||||
func updateBotHome(telegram *tg.TG, userID int64) {
|
||||
bot, err := newBot(telegram, userID)
|
||||
if err != nil {
|
||||
slog.Error("Bot error: can't create bot", "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
err = bot.ShowHome(nil)
|
||||
if err != nil {
|
||||
slog.Error("Server error: can't update bot home", "userID", userID, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// adjustTime adjusts the time in a given line by the specified number of hours
|
||||
func adjustTime(line string, hours int) string {
|
||||
// Regex to match time in format HH:MM
|
||||
re := regexp.MustCompile("^`\\d{2}:\\d{2}`")
|
||||
match := re.FindString(line)
|
||||
match = strings.Trim(match, "`")
|
||||
if match != "" {
|
||||
// Parse the time
|
||||
t, err := time.Parse("15:04", match)
|
||||
if err != nil {
|
||||
fmt.Println("Error parsing time:", err)
|
||||
return line
|
||||
}
|
||||
// Adjust the time by the specified number of hours
|
||||
t = t.Add(time.Duration(hours) * time.Hour)
|
||||
// Replace the time in the line with the new time
|
||||
return strings.Replace(line, match, t.Format("15:04"), 1)
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Println("Usage: go run main.go <file.md> <hours>")
|
||||
return
|
||||
}
|
||||
|
||||
// Get the file name and hours from command line arguments
|
||||
fileName := os.Args[1]
|
||||
hours, err := strconv.Atoi(os.Args[2])
|
||||
if err != nil {
|
||||
fmt.Println("Invalid number of hours:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Open the file
|
||||
file, err := os.Open(fileName)
|
||||
if err != nil {
|
||||
fmt.Println("Error opening file:", err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Create a scanner to read the file line by line
|
||||
scanner := bufio.NewScanner(file)
|
||||
var lines []string
|
||||
|
||||
// Process each line
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
adjustedLine := adjustTime(line, hours)
|
||||
lines = append(lines, adjustedLine)
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
fmt.Println("Error reading file:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Write the adjusted lines back to the file
|
||||
outputFile, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
fmt.Println("Error creating output file:", err)
|
||||
return
|
||||
}
|
||||
defer outputFile.Close()
|
||||
|
||||
writer := bufio.NewWriter(outputFile)
|
||||
for _, line := range lines {
|
||||
_, err := writer.WriteString(line + "\n")
|
||||
if err != nil {
|
||||
fmt.Println("Error writing to file:", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
writer.Flush()
|
||||
|
||||
fmt.Println("Time adjustment completed successfully.")
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// adjustTime adjusts the time in a given line by the specified number of hours
|
||||
func adjustTime(line string, hours int) string {
|
||||
// Regex to match time in format HH:MM
|
||||
re := regexp.MustCompile("^`\\d{2}:\\d{2}`")
|
||||
match := re.FindString(line)
|
||||
match = strings.Trim(match, "`")
|
||||
if match != "" {
|
||||
// Parse the time
|
||||
t, err := time.Parse("15:04", match)
|
||||
if err != nil {
|
||||
fmt.Println("Error parsing time:", err)
|
||||
return line
|
||||
}
|
||||
// Adjust the time by the specified number of hours
|
||||
t = t.Add(time.Duration(hours) * time.Hour)
|
||||
// Replace the time in the line with the new time
|
||||
return strings.Replace(line, match, t.Format("15:04"), 1)
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Println("Usage: go run main.go <file.md> <hours>")
|
||||
return
|
||||
}
|
||||
|
||||
// Get the file name and hours from command line arguments
|
||||
fileName := os.Args[1]
|
||||
hours, err := strconv.Atoi(os.Args[2])
|
||||
if err != nil {
|
||||
fmt.Println("Invalid number of hours:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Open the file
|
||||
file, err := os.Open(fileName)
|
||||
if err != nil {
|
||||
fmt.Println("Error opening file:", err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Create a scanner to read the file line by line
|
||||
scanner := bufio.NewScanner(file)
|
||||
var lines []string
|
||||
|
||||
// Process each line
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
adjustedLine := adjustTime(line, hours)
|
||||
lines = append(lines, adjustedLine)
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
fmt.Println("Error reading file:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Write the adjusted lines back to the file
|
||||
outputFile, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
fmt.Println("Error creating output file:", err)
|
||||
return
|
||||
}
|
||||
defer outputFile.Close()
|
||||
|
||||
writer := bufio.NewWriter(outputFile)
|
||||
for _, line := range lines {
|
||||
_, err := writer.WriteString(line + "\n")
|
||||
if err != nil {
|
||||
fmt.Println("Error writing to file:", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
writer.Flush()
|
||||
|
||||
fmt.Println("Time adjustment completed successfully.")
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
// Converts [[wikilinks]] to standard markdown [Link Name](/path/Link%20Name.md).
|
||||
// Scans all .md files in a directory first to resolve link targets by filename.
|
||||
//
|
||||
// Usage: go run ./cmd/tomdlinks <dir>
|
||||
//
|
||||
// go run ./cmd/tomdlinks --dry-run <dir>
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dryRun := false
|
||||
dir := "."
|
||||
|
||||
for _, arg := range os.Args[1:] {
|
||||
if arg == "--dry-run" {
|
||||
dryRun = true
|
||||
} else {
|
||||
dir = arg
|
||||
}
|
||||
}
|
||||
|
||||
// Scan all .md files and build a map: display name -> relative path
|
||||
candidates := map[string]string{}
|
||||
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, path)
|
||||
name := strings.TrimSuffix(filepath.Base(rel), ".md")
|
||||
candidates[name] = rel
|
||||
return nil
|
||||
})
|
||||
|
||||
wikiRE := regexp.MustCompile(`\[\[([^\[\]]+)\]\]`)
|
||||
|
||||
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
content := string(data)
|
||||
changed := false
|
||||
|
||||
result := wikiRE.ReplaceAllStringFunc(content, func(match string) string {
|
||||
name := match[2 : len(match)-2]
|
||||
target, ok := candidates[name]
|
||||
if !ok {
|
||||
return match
|
||||
}
|
||||
// Escape parens too - an unescaped ) in a path closes the
|
||||
// markdown link early (mirrors web's encodeLinkPath).
|
||||
url := "/" + strings.NewReplacer(" ", "%20", "(", "%28", ")", "%29").Replace(target)
|
||||
changed = true
|
||||
return fmt.Sprintf("[%s](%s)", name, url)
|
||||
})
|
||||
|
||||
if !changed {
|
||||
return nil
|
||||
}
|
||||
|
||||
rel, _ := filepath.Rel(dir, path)
|
||||
if dryRun {
|
||||
fmt.Printf("would update: %s\n", rel)
|
||||
return nil
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, []byte(result), info.Mode())
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error writing %s: %s\n", rel, err)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("updated: %s\n", rel)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Parses Whoop CSV exports and prints a 10-day journal summary.
|
||||
//
|
||||
// Usage: go run ./cmd/scripts/whoop <path-to-whoop-export-dir>
|
||||
// Example: go run ./cmd/scripts/whoop ./whoop
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type day struct {
|
||||
Date time.Time
|
||||
Sleep sleepData
|
||||
Recovery recoveryData
|
||||
Workouts []workout
|
||||
}
|
||||
|
||||
type sleepData struct {
|
||||
Performance int
|
||||
AsleepMin int
|
||||
Present bool
|
||||
}
|
||||
|
||||
type recoveryData struct {
|
||||
Score int
|
||||
HRV int
|
||||
RHR int
|
||||
Strain float64
|
||||
Present bool
|
||||
}
|
||||
|
||||
type workout struct {
|
||||
Activity string
|
||||
DurationMin int
|
||||
}
|
||||
|
||||
func main() {
|
||||
dir := "."
|
||||
if len(os.Args) > 1 {
|
||||
dir = os.Args[1]
|
||||
}
|
||||
|
||||
days := map[string]*day{}
|
||||
dayOf := func(t time.Time) *day {
|
||||
key := t.Format("2006-01-02")
|
||||
if d, ok := days[key]; ok {
|
||||
return d
|
||||
}
|
||||
d := &day{Date: t}
|
||||
days[key] = d
|
||||
return d
|
||||
}
|
||||
|
||||
// Parse sleeps - use wake onset date as the day, skip naps.
|
||||
parseSleeps(filepath.Join(dir, "sleeps.csv"), dayOf)
|
||||
|
||||
// Parse physiological cycles - recovery, HRV, RHR, strain.
|
||||
parseCycles(filepath.Join(dir, "physiological_cycles.csv"), dayOf)
|
||||
|
||||
// Parse workouts.
|
||||
parseWorkouts(filepath.Join(dir, "workouts.csv"), dayOf)
|
||||
|
||||
// Collect and sort days descending.
|
||||
var sorted []*day
|
||||
for _, d := range days {
|
||||
sorted = append(sorted, d)
|
||||
}
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
return sorted[i].Date.After(sorted[j].Date)
|
||||
})
|
||||
|
||||
// Print last 10 days.
|
||||
n := 10
|
||||
if len(sorted) < n {
|
||||
n = len(sorted)
|
||||
}
|
||||
for _, d := range sorted[:n] {
|
||||
printDay(d)
|
||||
fmt.Println()
|
||||
}
|
||||
}
|
||||
|
||||
func parseSleeps(path string, dayOf func(time.Time) *day) {
|
||||
rows := readCSV(path)
|
||||
if len(rows) < 2 {
|
||||
return
|
||||
}
|
||||
header := indexHeader(rows[0])
|
||||
|
||||
for _, row := range rows[1:] {
|
||||
if field(row, header, "Nap") == "true" {
|
||||
continue
|
||||
}
|
||||
|
||||
wake, err := parseTime(field(row, header, "Wake onset"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
d := dayOf(wake)
|
||||
d.Sleep.Present = true
|
||||
d.Sleep.Performance = atoi(field(row, header, "Sleep performance %"))
|
||||
d.Sleep.AsleepMin = atoi(field(row, header, "Asleep duration (min)"))
|
||||
}
|
||||
}
|
||||
|
||||
func parseCycles(path string, dayOf func(time.Time) *day) {
|
||||
rows := readCSV(path)
|
||||
if len(rows) < 2 {
|
||||
return
|
||||
}
|
||||
header := indexHeader(rows[0])
|
||||
|
||||
for _, row := range rows[1:] {
|
||||
wake, err := parseTime(field(row, header, "Wake onset"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
d := dayOf(wake)
|
||||
d.Recovery.Present = true
|
||||
d.Recovery.Score = atoi(field(row, header, "Recovery score %"))
|
||||
d.Recovery.HRV = atoi(field(row, header, "Heart rate variability (ms)"))
|
||||
d.Recovery.RHR = atoi(field(row, header, "Resting heart rate (bpm)"))
|
||||
d.Recovery.Strain = atof(field(row, header, "Day Strain"))
|
||||
}
|
||||
}
|
||||
|
||||
func parseWorkouts(path string, dayOf func(time.Time) *day) {
|
||||
rows := readCSV(path)
|
||||
if len(rows) < 2 {
|
||||
return
|
||||
}
|
||||
header := indexHeader(rows[0])
|
||||
|
||||
for _, row := range rows[1:] {
|
||||
start, err := parseTime(field(row, header, "Workout start time"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
d := dayOf(start)
|
||||
d.Workouts = append(d.Workouts, workout{
|
||||
Activity: field(row, header, "Activity name"),
|
||||
DurationMin: atoi(field(row, header, "Duration (min)")),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func printDay(d *day) {
|
||||
fmt.Printf("#### %d %s, %s\n", d.Date.Day(), d.Date.Format("January"), d.Date.Weekday())
|
||||
|
||||
if d.Sleep.Present {
|
||||
h := d.Sleep.AsleepMin / 60
|
||||
m := d.Sleep.AsleepMin % 60
|
||||
fmt.Printf("- Sleep: %d%%, %dh %02dm\n", d.Sleep.Performance, h, m)
|
||||
}
|
||||
|
||||
if d.Recovery.Present {
|
||||
fmt.Printf("- Recovery: %d%%, HRV %d, RHR %d\n", d.Recovery.Score, d.Recovery.HRV, d.Recovery.RHR)
|
||||
if d.Recovery.Strain > 0 {
|
||||
fmt.Printf("- Strain: %.1f\n", d.Recovery.Strain)
|
||||
}
|
||||
}
|
||||
|
||||
for _, w := range d.Workouts {
|
||||
fmt.Printf("- Workout: %s %dm\n", w.Activity, w.DurationMin)
|
||||
}
|
||||
}
|
||||
|
||||
// CSV helpers
|
||||
|
||||
func readCSV(path string) [][]string {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: %s\n", err)
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r := csv.NewReader(f)
|
||||
rows, err := r.ReadAll()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: can't parse %s: %s\n", path, err)
|
||||
return nil
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func indexHeader(row []string) map[string]int {
|
||||
m := make(map[string]int, len(row))
|
||||
for i, col := range row {
|
||||
m[strings.TrimSpace(col)] = i
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func field(row []string, header map[string]int, name string) string {
|
||||
i, ok := header[name]
|
||||
if !ok || i >= len(row) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(row[i])
|
||||
}
|
||||
|
||||
func parseTime(s string) (time.Time, error) {
|
||||
return time.Parse("2006-01-02 15:04:05", s)
|
||||
}
|
||||
|
||||
func atoi(s string) int {
|
||||
// Handle float strings like "421.0"
|
||||
if strings.Contains(s, ".") {
|
||||
f, _ := strconv.ParseFloat(s, 64)
|
||||
return int(f)
|
||||
}
|
||||
v, _ := strconv.Atoi(s)
|
||||
return v
|
||||
}
|
||||
|
||||
func atof(s string) float64 {
|
||||
v, _ := strconv.ParseFloat(s, 64)
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
services:
|
||||
files-md:
|
||||
build: .
|
||||
image: files-md
|
||||
ports:
|
||||
- "80:8080"
|
||||
# - "443:443"
|
||||
volumes:
|
||||
- storage:/app/storage
|
||||
- tokens:/app/tokens
|
||||
environment:
|
||||
APP_URL: http://localhost
|
||||
# API_URL: http://api.localhost # Uncomment if you want synchronization API
|
||||
STORAGE_DIR: /app/storage
|
||||
TOKENS_DIR: /app/tokens
|
||||
CERT_DIR: "" # Provide a dir if you want HTTPS enabled, e.g. /opt/certs
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
storage:
|
||||
tokens:
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
# Telegram bot
|
||||
|
||||
How the server wires Telegram updates into per-user workers and how `Bot.Reply` decides what to do with each message.
|
||||
|
||||
## High-level architecture
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
TG[Telegram API]
|
||||
PWA[PWA app.files.md]
|
||||
|
||||
subgraph Server [server - one binary]
|
||||
Main[main update loop<br/>api.GetUpdatesChan]
|
||||
Router{Route by userID}
|
||||
UCh[per-user channel]
|
||||
Sup[supervisor goroutine<br/>panic-recovering]
|
||||
Proc[processUserUpdates<br/>sequential loop]
|
||||
Bot[Bot.Reply]
|
||||
|
||||
Web[sync.Serve<br/>HTTP API]
|
||||
Worker[worker ticker 5s<br/>MoveDueTasks<br/>RemoveCompletedChecklistItems]
|
||||
|
||||
UserFS[(UserFS<br/>storage/userID/*.md)]
|
||||
DB[(per-user DB<br/>in-memory state)]
|
||||
Cfg[(config.json)]
|
||||
end
|
||||
|
||||
TG -->|long poll updates| Main
|
||||
Main --> Router
|
||||
Router -->|first time: spawn| Sup
|
||||
Router -->|every update| UCh
|
||||
UCh --> Sup
|
||||
Sup --> Proc
|
||||
Proc --> Bot
|
||||
|
||||
PWA <-->|POST /syncFilenames, /syncFile, /syncMediaFilenames, /syncMediaFile| Web
|
||||
|
||||
Worker -->|due tasks| Bot
|
||||
Bot -->|read/write .md| UserFS
|
||||
Bot -->|temp state| DB
|
||||
Bot -->|preferences| Cfg
|
||||
Web -->|read/write .md| UserFS
|
||||
Bot -->|send, edit, delete msgs| TG
|
||||
```
|
||||
|
||||
The server runs one binary with three long-running components:
|
||||
|
||||
- **Telegram update loop** (`cmd/server/server.go`) - long-polls Telegram, routes each update to a per-user goroutine. Per-user channels serialize one user's messages so concurrent edits to the same files can't race.
|
||||
- **HTTP sync server** (`server/sync`) - serves the PWA's sync requests (`/syncFilenames`, `/syncFile`, `/syncMediaFilenames`, `/syncMediaFile`). When the web app changes `Chat.md` or the chat, it calls `OnChatUpdate` which triggers the bot to send the user a fresh home keyboard so the two stay in lockstep.
|
||||
- **Worker ticker** - every 5 seconds moves scheduled tasks out of `later` into `chat`, and prunes completed checklist items.
|
||||
|
||||
Everything reads and writes the same per-user filesystem tree (`UserFS`), which is the single source of truth - `.md` files on disk. The PWA fetches those same files through the sync API.
|
||||
|
||||
## `Bot.Reply` - reply flow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Start([Update arrives at Bot.Reply])
|
||||
IQ{Inline query?}
|
||||
Plug{Plugin.CanHandle?}
|
||||
ViaBot{Sent via bot?<br/>inline result}
|
||||
Cmd{extractCmd<br/>returns a command?}
|
||||
IsCB{Callback query?}
|
||||
HasImg{Has image?}
|
||||
|
||||
Search[answerSearch<br/>return file results]
|
||||
ChanSave[addToFile<br/>append to ChannelName.md]
|
||||
PlugRun[plugin.Handle<br/>send output<br/>ShowHome]
|
||||
FileReq[answerFileRequest<br/>resolve file]
|
||||
DelKB[delAllKeyboards]
|
||||
Handler[dispatch to handlers map<br/>e.g. showMoveTo, moveToDir,<br/>complete, schedule, ...]
|
||||
AnswerCB[AnswerCallbackQuery<br/>completedMsg or empty]
|
||||
SaveImg[saveFromImage]
|
||||
SaveTxt[saveFromTextMsg]
|
||||
|
||||
Start --> IQ
|
||||
IQ -->|yes| Search --> End([return])
|
||||
IQ -->|no| Chan
|
||||
Chan -->|yes| ChanSave --> End
|
||||
Chan -->|no| Plug
|
||||
Plug -->|yes| PlugRun --> End
|
||||
Plug -->|no| ViaBot
|
||||
ViaBot -->|yes| FileReq --> End
|
||||
ViaBot -->|no| Cmd
|
||||
Cmd -->|yes, not callback| DelKB --> Handler
|
||||
Cmd -->|yes, callback| Handler
|
||||
Handler --> IsCB
|
||||
IsCB -->|yes| AnswerCB --> End
|
||||
IsCB -->|no| End
|
||||
Cmd -->|no| HasImg
|
||||
HasImg -->|yes| SaveImg --> End
|
||||
HasImg -->|no| SaveTxt --> End
|
||||
|
||||
style Search fill:#dfe,stroke:#374,color:#000
|
||||
style ChanSave fill:#dfe,stroke:#374,color:#000
|
||||
style PlugRun fill:#dfe,stroke:#374,color:#000
|
||||
style FileReq fill:#dfe,stroke:#374,color:#000
|
||||
style Handler fill:#ffd,stroke:#c80,color:#000
|
||||
style SaveImg fill:#fde,stroke:#a36,color:#000
|
||||
style SaveTxt fill:#fde,stroke:#a36,color:#000
|
||||
```
|
||||
|
||||
The decision is strictly top-to-bottom - the first matching case wins. Green terminals are read-only or side-channel responses; yellow is the large callback/command dispatch table; red is the save path for fresh user content.
|
||||
|
||||
### Main steps inside the save paths
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
subgraph txt [saveFromTextMsg]
|
||||
T1([message text]) --> T2[extractMarkdown]
|
||||
T2 --> T3{recent forward<br/>within collapse window?}
|
||||
T3 -->|yes| T4[createOrAdd to Chat.md] --> TEnd([return])
|
||||
T3 -->|no| T5{reply to a<br/>previous bot msg?}
|
||||
T5 -->|yes| T6[addToRepliedFile<br/>append to that note] --> TEnd
|
||||
T5 -->|no| T7[saveToInbox]
|
||||
T7 --> T8{ChatOnlyMode?}
|
||||
T8 -->|yes| T9[react 👌] --> TEnd
|
||||
T8 -->|no| T10{JournalOnlyMode?}
|
||||
T10 -->|yes| T11[moveToJournal] --> TEnd
|
||||
T10 -->|no| T12[showMoveTo<br/>buttons: to Home,<br/>to file, to dir, ...] --> TEnd
|
||||
end
|
||||
|
||||
subgraph img [saveFromImage]
|
||||
I1([photo/document]) --> I2[DownloadFile]
|
||||
I2 --> I3[fs.Write media/tg_*.ext]
|
||||
I3 --> I4[build markdown image link<br/>plus caption]
|
||||
I4 --> I5[same branching as saveFromTextMsg:<br/>collapse -> reply -> Inbox -> showMoveTo]
|
||||
end
|
||||
```
|
||||
|
||||
Both save paths converge on `saveToInbox` (append to `Inbox.md` with a timestamp) and then `showMoveTo`, which presents the user with action buttons. Picking a button fires a callback that re-enters `Bot.Reply`, hits the command branch, and runs the matching handler from the big map at `bot.go:327–415` (move to file, move to dir, schedule, complete, share, rename, etc.).
|
||||
|
||||
### What the handlers table looks like
|
||||
|
||||
A small taste of the command namespace (~90 entries in total, defined as `CmdX` constants around `bot.go:128–207`):
|
||||
|
||||
| Category | Examples |
|
||||
| --- | --- |
|
||||
| Views | `ShowHome`, `ShowLater`, `ShowFiles`, `ShowDirs`, `ShowChecklists`, `ShowSettings` |
|
||||
| Move | `MoveToExistingDir`, `MoveToNewFile`, `MoveToJournal`, `MoveToLater`, `MoveToChecklist`, `MoveToRead`/`Watch`/`Shop` |
|
||||
| Complete | `Complete`, `CompleteFromInbox`, `CompleteListItem`, `CompleteHabit` |
|
||||
| Schedule | `Schedule`, `ScheduleForTmrw`, `ShowScheduleForDay`, `Pomodoro` |
|
||||
| Rename | `ShowRename`, `ShowRenameFile`, `Rename` |
|
||||
| Settings | `TasksOnlyMode`, `NotesOnlyMode`, `JournalOnlyMode`, `FullMode`, `ChatMode`, `Timezone` |
|
||||
| Other | `OpenInApp`, `Download`, `Share`, `Help`, `Stats` |
|
||||
|
||||
Shortcut suffixes like ` jj` / ` жж` (append to journal) or `++` (append to most recently used file) are expanded into normal commands in `extractCmd` before dispatch.
|
||||
|
||||
## Concurrency guarantees in one line
|
||||
|
||||
One user's updates are processed strictly sequentially inside their own goroutine, but different users run in parallel - so the bot never races its own file writes for a single user, and the web app's sync API can safely modify the same files as the bot because the per-user worker holds the only write path for bot-initiated changes.
|
||||
@@ -0,0 +1,39 @@
|
||||
# End-to-end tests
|
||||
|
||||
## One-time setup
|
||||
|
||||
Create a `.env` file in the project root:
|
||||
|
||||
```
|
||||
APP_URL=http://localhost:3000
|
||||
API_URL=http://localhost:8080
|
||||
TOKENS_DIR="<PATH_TO_YOUR_FILES.MD_REPO/storage/-1"
|
||||
CERT_DIR=""
|
||||
TOKENS_SALT=""
|
||||
```
|
||||
|
||||
`APP_URL` is where Playwright serves the webapp (`tests/playwright.config.js` starts `npx http-server ../web -p 3000`). `API_URL` is where the Go server listens; the tests point the browser at it via `localStorage.setItem('apiUrl', ...)` in `setup()`.
|
||||
|
||||
## Commands
|
||||
|
||||
All commands are defined in the project root `Makefile`.
|
||||
|
||||
| Command | What it does |
|
||||
|---|---|
|
||||
| `make e2e` | Headless run of every spec. Kills any stray `server`, starts a fresh one in the background, then runs `npm run test` in `tests/`. |
|
||||
| `make e2e test="name"` | Same as above, but filters by test name (passed to Playwright as `-g "name"`). |
|
||||
| `make e2eh` | Headed run - same as `e2e` but opens a visible browser window (`npm run test:headed`). Good for debugging visually. |
|
||||
| `make e2eh test="name"` | Headed run filtered by name. |
|
||||
| `make e2es test="name"` | Single-test headless run **without restarting the server**. Assumes a server is already running. Useful when iterating on one test against an already-warm server. |
|
||||
| `make e2esh test="name"` | Same as `e2es` but headed. |
|
||||
|
||||
The `test="..."` value is a substring / regex matched against test titles (`test('…')` names in the spec files).
|
||||
|
||||
## Where test data lives
|
||||
|
||||
Every test-generated artifact goes under `storage/<WORKER_INDEX>/` at the project root. Playwright runs tests in parallel (8 workers by default; see `tests/playwright.config.js`), and each worker's server-side user storage is isolated by its worker ID:
|
||||
|
||||
- `storage/0/`, `storage/1/`, …, `storage/7/` - per-worker user filesystems. `tests/sync.spec.js` `beforeEach` wipes and recreates the current worker's directory on every test, so state never leaks between tests.
|
||||
- `storage/-1/` - shared tokens directory. Contains one file per worker, named by `sha256(workerIndex + salt)`, whose contents are the worker's user ID. Used by the server's token middleware to authenticate incoming sync requests from the test browser.
|
||||
|
||||
Feel free to delete `storage/` whenever - it's rebuilt on the next `beforeEach`.
|
||||
@@ -0,0 +1,163 @@
|
||||
# Sync flow
|
||||
|
||||
How `openFile`, `syncCurrentEditor`, `syncFilesWithServer`, and `syncLocalFileWithServer` hand off work, and how the server knows what to send or receive.
|
||||
|
||||
## Triggers and who calls whom
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
U[User: sidebar click, link click, popstate] -->|openFile| OF[openFile]
|
||||
Timer1[setInterval 1000ms saver] -->|syncCurrentEditor| SCE[syncCurrentEditor]
|
||||
Focus[focusin / focus event] -->|syncCurrentEditor| SCE
|
||||
Timer2[setInterval syncFilesWithServer + syncMediaFiles] -->|syncFilesWithServer| STS[syncFilesWithServer]
|
||||
|
||||
OF -->|save previous editor before swapping| SCE
|
||||
SCE -->|switchAwayEditor=false: end of fn| SLF[syncLocalFileWithServer]
|
||||
STS -->|per-file path| POST1[POST /syncFile per file]
|
||||
SLF --> POST2[POST /syncFile]
|
||||
|
||||
STS -->|batch modified/deleted| POST3[POST /syncFilenames]
|
||||
```
|
||||
|
||||
The red node is the drift-seal line (files.js:1028). If `currentEditor` was rotated by anything during the yellow await above, this write lands on the wrong editor instance.
|
||||
|
||||
## syncCurrentEditor - both flag branches
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Enter([syncCurrentEditor switchAwayEditor]) --> G1{files undef, debug,<br/>or path undefined?}
|
||||
G1 -->|yes| Ret1([return])
|
||||
G1 -->|no| G2{isSaving OR<br/>isMessingWithCurrentEditor?}
|
||||
G2 -->|yes| Ret2([return])
|
||||
G2 -->|no| SetFlag[isMessingWithCurrentEditor = true;<br/>path = currentEditor.path]
|
||||
|
||||
SetFlag --> IsInbox{path == INBOX_PATH?}
|
||||
IsInbox -->|yes| InboxBranch[chat sync logic]
|
||||
InboxBranch --> Ret3([return])
|
||||
|
||||
IsInbox -->|no| RenameCheck{firstLine's filename<br/>!= current filename?}
|
||||
RenameCheck -->|yes| Rename[await remove path<br/>await getFileHandle newPath<br/>await writeIfContentIsDifferent<br/>await renderSidebar]
|
||||
Rename --> Ret4([return])
|
||||
RenameCheck -->|no| DiffCheck[await isContentEqual<br/>path, getCurrentContent]
|
||||
|
||||
DiffCheck --> SameGuard{isCurrentEditorSame?}
|
||||
SameGuard -->|no| Ret5([return])
|
||||
SameGuard -->|yes| ModCheck{contentWasModifiedLocally<br/>AND editor.isClean?}
|
||||
|
||||
ModCheck -->|yes| SwitchGate{switchAwayEditor?}
|
||||
SwitchGate -->|true: skip reload| Clear[isMessingWithCurrentEditor = false]
|
||||
SwitchGate -->|false: reload from disk| Reload[await openFile path, false]
|
||||
Reload --> Clear
|
||||
|
||||
ModCheck -->|editor dirty| Save[write editor content to disk;<br/>editor.markClean]
|
||||
Save --> Clear
|
||||
ModCheck -->|neither| Clear
|
||||
|
||||
Clear --> ServerGate{switchAwayEditor?}
|
||||
ServerGate -->|true: skip server sync| Done([return])
|
||||
ServerGate -->|false| PushServer[await syncLocalFileWithServer path]
|
||||
PushServer --> Done
|
||||
|
||||
style Rename fill:#faa,stroke:#900,color:#000
|
||||
style Reload fill:#fca,stroke:#c80,color:#000
|
||||
style SwitchGate fill:#cfc,stroke:#070,color:#000
|
||||
style ServerGate fill:#cfc,stroke:#070,color:#000
|
||||
```
|
||||
|
||||
The two green gates are the `switchAwayEditor` branches. The orange `Reload` is the one we neutralised (it used to recurse into `openFile` without an `el` arg and clobber the main editor). The red `Rename` block is the executioner that actually deletes and creates files on disk - still live, fires whenever first-line header disagrees with filename.
|
||||
|
||||
## Sync with the server - batch vs per-file
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client
|
||||
participant Server
|
||||
|
||||
Note over Client: syncFilesWithServer fires
|
||||
Client->>Client: collect modified and deleted files (skip editor and editor2 paths)
|
||||
Client->>Server: POST /syncFilenames with modified, deleted, timestamps
|
||||
Server-->>Client: files, timestamps, renames
|
||||
Client->>Client: write non-current files to disk and update server.files snapshot
|
||||
Client->>Client: advance per-dir timestamp pointers
|
||||
|
||||
Note over Client: syncCurrentEditor finishes the switchAwayEditor=false branch
|
||||
Client->>Client: syncLocalFileWithServer for the active editor
|
||||
Client->>Server: POST /syncFile with path, lastModified, clientLastModified, clientLastSynced, content
|
||||
alt notModified
|
||||
Server-->>Client: notModified
|
||||
Client->>Client: advance lastClientModified only
|
||||
else updatedOnServer
|
||||
Server-->>Client: updatedOnServer with new lastModified
|
||||
Client->>Client: record the server lastModified, no disk write
|
||||
else merged or ok
|
||||
Server-->>Client: content and lastModified
|
||||
Client->>Client: writeIfContentIsDifferent, then openFile if path matches editor.path
|
||||
end
|
||||
```
|
||||
|
||||
### How the server knows there's something to sync
|
||||
|
||||
Two mechanisms, running in parallel:
|
||||
|
||||
1. **Batch: `syncFilesWithServer` → `POST /syncFilenames`.** The client sends:
|
||||
- `modified`: files whose disk `lastModified` is newer than the `lastClientSynced` pointer recorded in `server.files` for that path.
|
||||
- `deleted`: files present in the client's `server.files` snapshot but no longer on disk.
|
||||
- `timestamps`: a per-directory pointer telling the server "everything I've seen up to here." The server replies with files newer than each directory's pointer. **The two currently-open editor files are skipped on both send and receive** (`files.js:230` and `files.js:577`) - they're handled by the per-file path instead, to avoid racing with the user's active edits.
|
||||
|
||||
2. **Per-file: `syncLocalFileWithServer` → `POST /syncFile`.** Called at the end of each `syncCurrentEditor` (when `switchAwayEditor=false`). Sends the single file's content plus its `lastModified` + `clientLastModified` + `clientLastSynced`. The server compares timestamps and responds with one of four statuses that the client maps to either "advance pointers only" or "write this content to disk."
|
||||
|
||||
The client's `server.files` object holds the triple `(content, lastModified, lastClientModified)` per path - this is the client's view of what the server thinks the world looks like, and the basis for deciding which files to include in the next `modified`/`deleted` lists. Persisted to `localStorage` under `SERVER_STORAGE_KEY`.
|
||||
|
||||
### Auth gate: `lastServerOk`
|
||||
|
||||
The auth token lives in an HttpOnly cookie, so JS can't see it directly. Instead, every successful response from the server stamps `localStorage.lastServerOk` with `Date.now()` via `markServerOk()` (files.js). `hasLastServerOk()` returns true if that key exists - which it only can if the server has previously accepted us. Use this as the gate before kicking off sync work: no stamp ⇒ no token ⇒ skip the request entirely. The flag is set in:
|
||||
|
||||
- `app.js` after the `/issuePermanentToken` exchange returns 200
|
||||
- `post()` after a 2xx response (covers all `/syncFilenames`, `/syncFile`, `/syncMediaFilenames`, `/syncMediaFile` upload calls now that they go through this helper)
|
||||
- `syncMediaFiles` directly after the raw `POST /syncMediaFile` download (binary blob, can't share `post()`)
|
||||
|
||||
If the server later 401s, the stamp stays - but the request will simply fail and no sync state advances, so we don't need to clear it.
|
||||
|
||||
## File deletion propogation across clients
|
||||
|
||||
A delete on one device has to travel through the server and reach every other device that still holds the file. The mechanism is an append-only `fslog` on disk: every server-side `userFS.Del` writes a `<ts> del <abs-path>` row, and every `/syncFilenames` response carries the deletes a given client hasn't seen yet.
|
||||
|
||||
### Why we need this log at all
|
||||
|
||||
Without it, the server only knows what currently exists on disk - it has no memory of what *used* to exist. Sync responses only list present files. So Client B, which still holds the deleted file locally, would see "this path is on my disk but not in the server response" and conclude it's a *new* local file → it would re-upload `foo.md` and the file resurrects. The fslog gives the server a memory of deletions, so it can tell B "yes, this used to exist, but it was deleted at time T - drop your stale copy."
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
autonumber
|
||||
participant A as Client A
|
||||
participant S as Server
|
||||
participant L as fslog<br/>(append-only file)
|
||||
participant B as Client B
|
||||
|
||||
Note over A,B: Steady state: both clients hold foo.md locally,<br/>server has foo.md on disk
|
||||
|
||||
rect rgb(245, 240, 230)
|
||||
Note over A: User deletes foo.md in the PWA
|
||||
A->>A: moveFile("/foo.md", "/archive/foo.md")<br/>(local FS only)
|
||||
A->>S: POST /syncFilenames<br/>{ deleted: ["/foo.md"],<br/> modified: [{path:"/archive/foo.md", ...}],<br/> serverTime: <A's cursor> }
|
||||
S->>S: userFS.Del("foo.md")<br/>removes from disk
|
||||
S->>L: append "<now> del /app/storage/<uid>/foo.md"
|
||||
S->>S: deletes = DeletesLog(uid, req.serverTime+1)<br/>→ {"foo.md": <now>}
|
||||
S->>S: suppress echo: drop entries that<br/>match request.Deleted
|
||||
S->>S: write /archive/foo.md
|
||||
S-->>A: response.deleted = {} (A's own delete<br/>was filtered)<br/>response.files = [...]
|
||||
end
|
||||
|
||||
Note over A,B: ...time passes, B opens app or hits sync interval...
|
||||
|
||||
rect rgb(230, 240, 245)
|
||||
B->>S: POST /syncFilenames<br/>{ deleted: [], modified: [...],<br/> serverTime: <B's cursor, before A's delete> }
|
||||
S->>L: scan fslog for this user
|
||||
S->>S: deletes = DeletesLog(uid, req.serverTime+1)<br/>→ {"foo.md": <ts of A's delete>}<br/>(no suppression: B didn't delete it)
|
||||
S-->>B: response.deleted = {"foo.md": <ts>}<br/>response.files = [archive/foo.md, ...]
|
||||
B->>B: for each (path, deletedAt) in response.deleted:<br/>local = getMemFile(path)<br/>if local && local.lastModified ≤ deletedAt:<br/> await remove(path)#59; removeServerFile(path)
|
||||
B->>B: write archive/foo.md from response.files
|
||||
end
|
||||
|
||||
Note over A,B: Both clients converged:<br/>foo.md gone, archive/foo.md present
|
||||
```
|
||||
@@ -0,0 +1,126 @@
|
||||
# Run your own server
|
||||
|
||||
## Containerized deployment (Docker/Podman)
|
||||
|
||||
Build and start:
|
||||
```bash
|
||||
$ docker compose up
|
||||
```
|
||||
|
||||
### Enable HTTPS
|
||||
In `compose.yaml`: set `CERT_DIR` to a persistent path, uncomment the `"443:443"` port.
|
||||
|
||||
|
||||
## Deploy on your own server (manual)
|
||||
|
||||
Install [Go](https://go.dev/doc/install) on your host machine.
|
||||
|
||||
Initialize server with folders and systemd service. Tested on Debian-based systems:
|
||||
```bash
|
||||
$ make init_server host=user@example.com salt=$(head -c 32 /dev/urandom | base64)
|
||||
```
|
||||
|
||||
Configure the `/app/.env` file:
|
||||
```
|
||||
BOT_API_TOKEN=<TELEGRAM_API_TOKEN_IF_NEEDED>
|
||||
STORAGE_DIR=/app/storage
|
||||
CERT_DIR=/opt/files.md
|
||||
TOKENS_DIR=/opt/files.md/tokens
|
||||
LOG_FILE=server.log
|
||||
API_URL=https://api.yourdomain.com
|
||||
APP_URL=https://app.youdomain.com
|
||||
```
|
||||
|
||||
Deploy a systemd service:
|
||||
```bash
|
||||
$ make deploy_systemd host=<YOUR_SSH_HOST>
|
||||
```
|
||||
|
||||
That's all :)
|
||||
|
||||
## Run your own Telegram Bot
|
||||
1) Install [Go](https://go.dev/doc/install)
|
||||
2) Register new telegram bot via [@BotFather](https://t.me/BotFather)
|
||||
3) Add `BOT_API_TOKEN=<YOUR_TELEGRAM_API_TOKEN>` line to `.env` file
|
||||
4) Redeploy/relaunch the server
|
||||
|
||||
Bot's artifacts can be seen in `./storage/<USER_ID>` folder.
|
||||
|
||||
## Linking a new device
|
||||
1) Open telegram bot
|
||||
2) Open `/app`
|
||||
3) Open the link in your browser
|
||||
4) Device is now linked
|
||||
|
||||
### Additional bot's settings
|
||||
1) For search functionality, enable `Inline Mode` for your bot in [@BotFather](https://t.me/BotFather)
|
||||
2) Press "Edit Commands", and send the following list:
|
||||
```
|
||||
chat - 🏠 Home
|
||||
files - 📄 Files
|
||||
dirs - 🗂 Dirs
|
||||
checklists - ☑️ Checklists
|
||||
schedule - 📆 Schedule
|
||||
postpone - 🦥 Postpone
|
||||
rename - ✏️ Rename
|
||||
move - ➡️ Move
|
||||
app - 🔗 Open in app
|
||||
settings - ⚙️ Settings
|
||||
help - 📕 Help
|
||||
```
|
||||
|
||||
## Hosting the bot on you local computer
|
||||
You can host the bot locally, because it doesn't expose any ports to the outside world (if you don't use habits functionality).
|
||||
It communicates with Telegram using pull API.
|
||||
|
||||
Create a symlink to your local folder with `.md` files for convenience:
|
||||
`ln -s <YOUR_EXISTING_DIR_WITH_MD_FILES> storage/<USER_ID>`
|
||||
|
||||
## Transfer files to another server
|
||||
|
||||
1) Backup your data (`/app/storage`)
|
||||
2) Be sure that all client app fully synced with the server (bring the app in the focus)
|
||||
3) Stop bot on old server, so no new files would be created.
|
||||
4) Compress all the files on one server: `tar -czvf storage.tar.gz storage`
|
||||
5) `scp` the file to your host machine: `scp SSH_HOST:/app/storage.tar.gz .`
|
||||
6) `scp` the file to your target machine
|
||||
|
||||
Synchronization is relying on `mtime`, so after compressing/decompressing the flag wouldn't be lost.
|
||||
|
||||
1) `cd /opt/files.md`
|
||||
2) `tar -czvf tokens.tar.gz tokens`
|
||||
3) `scp` to same dir on target machine
|
||||
|
||||
We don't need to transfer fslog (renames), if we're certain that all clients read the log.
|
||||
|
||||
1) Extract all files on new server
|
||||
2) Transfer `BOT_API_TOKEN`
|
||||
3) Launch server
|
||||
4) Execute `localStorage.setItem('ApiHost', 'YOUR_NEW_API_HOST');` in your PWA applications
|
||||
5) Make sure that all files are available
|
||||
6) Cleanup the oldserver
|
||||
|
||||
## Maintenance notes
|
||||
Add this to your crontab (`crontab -e`) for daily git backups:
|
||||
`0 0 * * * cd /app/storage/<YOUR_TELEGRAM_ID> && git add . && git commit -m "$(date +\%d.\%m.\%Y)"`
|
||||
|
||||
Execute `git init` in your folder before that, to init a git repository.
|
||||
|
||||
If you have non-ASCI character in filenames, disable quoting:
|
||||
`git config --global core.quotePath false`
|
||||
|
||||
Systemd journal:
|
||||
`sudo journalctl -u filesmd`
|
||||
|
||||
Find forbidden character in filenames (can be executed in user's storage folder):
|
||||
`find . -name '*[<>:"|\?*]*'`
|
||||
|
||||
Remove forbidden filename characters:
|
||||
```bash
|
||||
find . -type f -name '*[<>:"|\?*]*' -print0 | while IFS= read -r -d '' f; do
|
||||
dir=$(dirname "$f")
|
||||
base=$(basename "$f")
|
||||
newbase="${base//[<>:\"|\\?*]/}"
|
||||
[ "$base" != "$newbase" ] && [ -n "$newbase" ] && mv -n -- "$f" "$dir/$newbase"
|
||||
done
|
||||
```
|
||||
@@ -0,0 +1,26 @@
|
||||
module github.com/zakirullin/files.md
|
||||
|
||||
go 1.24
|
||||
|
||||
require (
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.2-0.20221020003552-4126fa611266
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/kelseyhightower/envconfig v1.4.0
|
||||
github.com/robfig/cron/v3 v3.0.0
|
||||
github.com/spf13/afero v1.9.5
|
||||
github.com/stretchr/testify v1.8.4
|
||||
golang.org/x/crypto v0.23.0
|
||||
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
|
||||
golang.org/x/net v0.25.0 // indirect
|
||||
golang.org/x/text v0.16.0
|
||||
)
|
||||
|
||||
require github.com/rivo/uniseg v0.4.7
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,466 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
|
||||
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
|
||||
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
|
||||
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
|
||||
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
|
||||
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
|
||||
cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.2-0.20221020003552-4126fa611266 h1:B1MTo1Xwp/SNvUOGxo7E95vIDXRYIJyF787suIZq9mU=
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.2-0.20221020003552-4126fa611266/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
|
||||
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
|
||||
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
|
||||
github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
|
||||
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
||||
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
|
||||
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
|
||||
google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
|
||||
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
|
||||
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
|
||||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
|
||||
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
+2901
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,84 @@
|
||||
// Don't want to put this hacky complex code into main bot file.
|
||||
// The purpose of this file is to collapse a few consecutive incoming messages
|
||||
// into one file. This is useful when a user forwards a few messages to the bot.
|
||||
//
|
||||
// 1) For a single message the flow is the same, no changes are made.
|
||||
// 2) For any additional messages, we check the time of the last message.
|
||||
// 3) If it was less than or equal to 1 second ago, we collapse it with the first message in the batch.
|
||||
// 4) A batch is a sequence of messages with a distance of no more than 1 second between them.
|
||||
// 5) That’s it.
|
||||
//
|
||||
// Suppose we have the following timestamps for incoming messages: 0 0 1 1 1 2 2 2 2 3.
|
||||
// This is all one batch (distance of no more than 1 second between them).
|
||||
// So we collapse all messages into the first message of the batch (time=0).
|
||||
//
|
||||
// Physically, a user cannot send so many messages manually, so we hypothesize
|
||||
// that these messages were forwarded. I don’t particularly like this assumption,
|
||||
// but that's the ugliness of real-world problems reflected into code.
|
||||
// When a user forwards a few messages from "Saved Messages" dialog
|
||||
// the messages don't have "is_forwarded" flag, so we can only
|
||||
// distinguish them by using this time-based heuristic.
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
firstMsgHashes sync.Map
|
||||
firstMsgTimes sync.Map
|
||||
)
|
||||
|
||||
func firstMsgTime(userID int64) (int, bool) {
|
||||
time, ok := firstMsgTimes.Load(userID)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return time.(int), true
|
||||
}
|
||||
|
||||
func setFirstMsgTime(userID int64, time int) {
|
||||
firstMsgTimes.Store(userID, time)
|
||||
}
|
||||
|
||||
func firstMsgHash(userID int64) (string, bool) {
|
||||
msg, ok := firstMsgHashes.Load(userID)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return msg.(string), true
|
||||
}
|
||||
|
||||
// setFirstMsgHash records the hash of the first entry in a forward batch.
|
||||
// If a batch is already active (within 1-second distance) this is a no-op —
|
||||
// only the first message's identity is tracked.
|
||||
func setFirstMsgHash(userID int64, msgHash string, time int) {
|
||||
firstTime, ok := firstMsgTime(userID)
|
||||
if ok {
|
||||
diff := time - firstTime
|
||||
// Sent in exactly same second or second after
|
||||
if diff == 0 || diff == 1 {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
firstMsgHashes.Store(userID, msgHash)
|
||||
}
|
||||
|
||||
func collapseToMsg(userID int64, time int) (string, bool) {
|
||||
firstTime, ok := firstMsgTime(userID)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
|
||||
diff := time - firstTime
|
||||
// Sent in exactly same second or second after
|
||||
if diff == 0 || diff == 1 {
|
||||
return firstMsgHash(userID)
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
"github.com/zakirullin/files.md/server/i18n"
|
||||
"github.com/zakirullin/files.md/server/pkg/tg"
|
||||
"github.com/zakirullin/files.md/server/pkg/txt"
|
||||
)
|
||||
|
||||
const (
|
||||
addBtn = "➕"
|
||||
delBtn = "➖"
|
||||
)
|
||||
|
||||
var AvailableMoveToBtns = []tg.Btn{
|
||||
tg.NewBtn(i18n.StrToTomorrow, tg.NewCmd(CmdScheduleForTmrw, nil)),
|
||||
tg.NewBtn(i18n.StrToLater, tg.NewCmd(CmdMoveToLater, nil)),
|
||||
tg.NewBtn(i18n.StrToADay, tg.NewCmd(CmdShowScheduleForDay, nil)),
|
||||
tg.NewBtn(i18n.StrToFile, tg.NewCmd(CmdShowMoveToDirOrFile, nil)),
|
||||
tg.NewBtn(i18n.StrToJournal, tg.NewCmd(CmdMoveToJournal, nil)),
|
||||
tg.NewBtn(i18n.StrToRead, tg.NewCmd(CmdMoveToRead, nil)),
|
||||
tg.NewBtn(i18n.StrToWatch, tg.NewCmd(CmdMoveToWatch, nil)),
|
||||
tg.NewBtn(i18n.StrToShop, tg.NewCmd(CmdMoveToShop, nil)),
|
||||
tg.NewBtn(i18n.StrToChecklist, tg.NewCmd(CmdShowMoveToChecklist, nil)),
|
||||
}
|
||||
|
||||
var AvailableQuickBtns = []tg.Btn{
|
||||
tg.NewBtn("Later", tg.NewCmd(CmdLater, nil)),
|
||||
tg.NewBtn("Search", tg.NewCustomCmd(CmdInlineQuerySearchEveryWhere, nil, tg.CmdTypeInlineQueryCurrentChat)),
|
||||
tg.NewBtn("Files", tg.NewCmd(CmdShowFiles, nil)),
|
||||
tg.NewBtn("Checklists", tg.NewCmd(CmdShowChecklists, nil)),
|
||||
tg.NewBtn("Postpone", tg.NewCmd(CmdShowPostpone, nil)),
|
||||
tg.NewBtn("Read", tg.NewCmd(CmdShowReadChecklist, nil)),
|
||||
tg.NewBtn("Watch", tg.NewCmd(CmdShowWatchChecklist, nil)),
|
||||
tg.NewBtn("Shop", tg.NewCmd(CmdShowShopChecklist, nil)),
|
||||
tg.NewBtn("Schedule", tg.NewCmd(CmdShowSchedule, nil)),
|
||||
tg.NewBtn("Habits", tg.NewCustomCmd(CmdWebAppHabits, nil, tg.CmdTypeWebApp)),
|
||||
tg.NewBtn("Random", tg.NewCmd(CmdRandomNote, nil)),
|
||||
}
|
||||
|
||||
func (b *Bot) showSettings(_ []string) error {
|
||||
var kb tg.Keyboard
|
||||
kb.AddRow(tg.NewBtn(txt.Emoji(i18n.Emoji("brain"), b.tr("Full mode")), tg.NewCmd(CmdFullMode, nil)))
|
||||
kb.AddRow(tg.NewBtn(txt.Emoji(i18n.Emoji("notes"), b.tr("Notes mode")), tg.NewCmd(CmdNotesOnlyMode, nil)))
|
||||
kb.AddRow(tg.NewBtn(txt.Emoji(i18n.Emoji("tasks"), b.tr("Tasks mode")), tg.NewCmd(CmdTasksOnlyMode, nil)))
|
||||
//kb.AddRow(tg.NewBtn(txt.Emoji(i18n.Emoji("journal"), b.tr("Journal mode")), tg.NewCmd(CmdJournalOnlyMode, nil)))
|
||||
kb.AddRow(tg.NewBtn("-", tg.NewCmd(CmdDoNothing, nil)))
|
||||
kb.AddRow(tg.NewBtn(i18n.StrQuickBtns, tg.NewCmd(CmdShowQuickBtnsSettings, nil)))
|
||||
kb.AddRow(tg.NewBtn(i18n.StrMoveToBtns, tg.NewCmd(CmdShowMoveToBtnsSettings, nil)))
|
||||
kb.AddRow(tg.NewBtn(txt.Emoji(i18n.Emoji("world"), b.tr("Timezone")), tg.NewCmd(CmdShowTimezone, nil)))
|
||||
kb.AddRow(tg.NewBtn(i18n.StrHome, tg.NewCmd(CmdShowHome, nil)))
|
||||
|
||||
err := b.showHTML("Settings:", &kb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("showSettings : %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Bot) showTimezone(_ []string) error {
|
||||
var kb tg.Keyboard
|
||||
timezones := []string{
|
||||
"UTC",
|
||||
|
||||
// Europe
|
||||
"Europe/Belgrade",
|
||||
"Europe/Berlin",
|
||||
"Europe/Istanbul",
|
||||
"Europe/London",
|
||||
"Europe/Madrid",
|
||||
"Europe/Moscow",
|
||||
"Europe/Paris",
|
||||
"Europe/Podgorica",
|
||||
"Europe/Warsaw",
|
||||
|
||||
// Asia
|
||||
"Asia/Nicosia",
|
||||
"Asia/Tbilisi",
|
||||
"Asia/Tokyo",
|
||||
|
||||
// Africa
|
||||
"Africa/Cairo",
|
||||
"Africa/Johannesburg",
|
||||
|
||||
// Americas
|
||||
"America/Buenos_Aires",
|
||||
"America/Lima",
|
||||
"America/Los_Angeles",
|
||||
"America/New_York",
|
||||
"America/Santiago",
|
||||
}
|
||||
|
||||
for _, tz := range timezones {
|
||||
name := tz
|
||||
if b.cfg.Timezone().String() == tz {
|
||||
name = "✅ " + name
|
||||
}
|
||||
kb.AddRow(tg.NewBtn(name, tg.NewCmd(CmdSetTimezone, []string{tz})))
|
||||
}
|
||||
kb.AddRow(tg.NewBtn(i18n.StrHome, tg.NewCmd(CmdShowHome, nil)))
|
||||
|
||||
err := b.showHTML("Timezone:", &kb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("showTimezone : %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Bot) setTimezone(params []string) error {
|
||||
timezone := params[0]
|
||||
|
||||
err := b.cfg.SetTimezone(timezone)
|
||||
if err != nil {
|
||||
return fmt.Errorf("setTimezone : %w", err)
|
||||
}
|
||||
|
||||
return b.ShowHome(nil)
|
||||
}
|
||||
|
||||
func (b *Bot) showQuickBtnsSettings(params []string) error {
|
||||
var kb tg.Keyboard
|
||||
|
||||
// Step 1. Append all buttons that are already chosen by user
|
||||
var usedCmds []string
|
||||
|
||||
// We iterate through hardcoded panel to preserve order of buttons in UI
|
||||
cmds, err := b.cfg.QuickCmds()
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't get quick cmds: %w", err)
|
||||
}
|
||||
|
||||
for _, cmd := range cmds {
|
||||
for _, btn := range AvailableQuickBtns {
|
||||
if btn.Cmd.Name != cmd {
|
||||
continue
|
||||
}
|
||||
|
||||
name := fmt.Sprintf("%s %s %s", i18n.Emoji(btn.Name), btn.Name, delBtn)
|
||||
enabledCmd := tg.NewCmd(CmdDelFromQuickBtns, []string{btn.Cmd.Name})
|
||||
kb.AddRow(tg.NewBtn(name, enabledCmd))
|
||||
usedCmds = append(usedCmds, cmd)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
kb.AddRow(tg.NewBtn("-", tg.NewCmd(CmdDoNothing, nil)))
|
||||
|
||||
// Step 2. now, let's fill buttons that are not disabled...
|
||||
for _, btn := range AvailableQuickBtns {
|
||||
// Check if command is enabled
|
||||
cmdUsed := false
|
||||
for _, usedCmd := range usedCmds {
|
||||
if btn.Cmd.Name == usedCmd {
|
||||
cmdUsed = true
|
||||
}
|
||||
}
|
||||
if cmdUsed {
|
||||
continue
|
||||
}
|
||||
// Command is not enabled, so add it to disabled list
|
||||
name := fmt.Sprintf("%s %s %s", i18n.Emoji(btn.Name), btn.Name, addBtn)
|
||||
disabledCmd := tg.NewCmd(CmdAddToQuickBtns, []string{btn.Cmd.Name})
|
||||
kb.AddRow(tg.NewBtn(name, disabledCmd))
|
||||
}
|
||||
|
||||
kb.AddRow(tg.NewBtn(i18n.StrHome, tg.NewCmd(CmdShowHome, nil)))
|
||||
|
||||
text := fmt.Sprintf("Configure quick buttons (%s = add to quick buttons, %s = to remove from quick buttons):", addBtn, delBtn)
|
||||
err = b.showHTML(text, &kb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("configureQuickPanel : %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Bot) addToQuickBtns(params []string) error {
|
||||
cmd := params[0]
|
||||
|
||||
// Search whether a command is valid
|
||||
found := false
|
||||
for _, btn := range AvailableQuickBtns {
|
||||
if btn.Cmd.Name == cmd {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
return fmt.Errorf("unknown command: %s", cmd)
|
||||
}
|
||||
|
||||
if cmd == "Habits" {
|
||||
b.fs.CreateDirsIfNotExist(fs.DirHabits, fs.DirInsights)
|
||||
}
|
||||
|
||||
err := b.cfg.AddQuickCmd(cmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't add to quick buttons: %w", err)
|
||||
}
|
||||
|
||||
return b.showQuickBtnsSettings([]string{})
|
||||
}
|
||||
|
||||
func (b *Bot) delFromQuickBtns(params []string) error {
|
||||
cmd := params[0]
|
||||
|
||||
_ = b.cfg.DelQuickCmd(cmd)
|
||||
|
||||
return b.showQuickBtnsSettings([]string{})
|
||||
}
|
||||
|
||||
func (b *Bot) quickBtns() []tg.Btn {
|
||||
quickBtnsRow := tg.NewRow()
|
||||
// We can tolerate missing quick btns
|
||||
cmds, _ := b.cfg.QuickCmds()
|
||||
for _, cmd := range cmds {
|
||||
for _, btn := range AvailableQuickBtns {
|
||||
if btn.Cmd.Name == cmd {
|
||||
if btn.Cmd.Name == CmdWebAppHabits {
|
||||
habitsUrl := fmt.Sprintf("%s/habits_v2/%d", config.ServerCfg.APIURL, b.userID)
|
||||
btn.Cmd.Params = []string{habitsUrl}
|
||||
}
|
||||
btn.Name = i18n.Emoji(btn.Name)
|
||||
|
||||
quickBtnsRow = append(quickBtnsRow, btn)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return quickBtnsRow
|
||||
}
|
||||
|
||||
// A little copy-paste from showQuickBtnsSettings
|
||||
func (b *Bot) showMoveToBtnsSettings(params []string) error {
|
||||
var kb tg.Keyboard
|
||||
|
||||
// Step 1. Append all buttons that are already chosen by user
|
||||
var usedCmds []string
|
||||
|
||||
// We iterate through hardcoded panel to preserve order of buttons in UI
|
||||
cmds, err := b.cfg.MoveToCmds()
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't get move to cmds: %w", err)
|
||||
}
|
||||
for _, cmd := range cmds {
|
||||
for _, btn := range AvailableMoveToBtns {
|
||||
if btn.Cmd.Name != cmd {
|
||||
continue
|
||||
}
|
||||
|
||||
name := txt.Emoji(delBtn, btn.Name)
|
||||
enabledCmd := tg.NewCmd(CmdDelFromMoveToBtns, []string{btn.Cmd.Name})
|
||||
kb.AddRow(tg.NewBtn(name, enabledCmd))
|
||||
usedCmds = append(usedCmds, cmd)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
kb.AddRow(tg.NewBtn("-", tg.NewCmd(CmdDoNothing, nil)))
|
||||
|
||||
// Step 2. now, let's fill buttons that are not disabled...
|
||||
for _, btn := range AvailableMoveToBtns {
|
||||
// Check if command is enabled
|
||||
cmdUsed := false
|
||||
for _, usedCmd := range usedCmds {
|
||||
if btn.Cmd.Name == usedCmd {
|
||||
cmdUsed = true
|
||||
}
|
||||
}
|
||||
if cmdUsed {
|
||||
continue
|
||||
}
|
||||
// Command is not enabled, so add it to disabled list
|
||||
name := txt.Emoji(addBtn, btn.Name)
|
||||
disabledCmd := tg.NewCmd(CmdAddToMoveToBtns, []string{btn.Cmd.Name})
|
||||
kb.AddRow(tg.NewBtn(name, disabledCmd))
|
||||
}
|
||||
|
||||
kb.AddRow(tg.NewBtn(i18n.StrHome, tg.NewCmd(CmdShowHome, nil)))
|
||||
|
||||
text := fmt.Sprintf("Configure quick panel (%s = add to panel, %s = to remove):", addBtn, delBtn)
|
||||
err = b.showHTML(text, &kb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("configureQuickPanel : %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Bot) addToMoveToBtns(params []string) error {
|
||||
cmd := params[0]
|
||||
|
||||
err := b.cfg.AddMoveToCmd(cmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't add to move to buttons: %w", err)
|
||||
}
|
||||
|
||||
return b.showMoveToBtnsSettings([]string{})
|
||||
}
|
||||
|
||||
func (b *Bot) delFromMoveToBtns(params []string) error {
|
||||
cmd := params[0]
|
||||
|
||||
err := b.cfg.DelMoveToCmd(cmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("button doesn't exist in user's prefs: %s", params[0])
|
||||
}
|
||||
|
||||
return b.showMoveToBtnsSettings([]string{})
|
||||
}
|
||||
|
||||
func (b *Bot) moveToBtns(msgHash string) []tg.Btn {
|
||||
moveToBtns := tg.NewRow()
|
||||
|
||||
cmds, err := b.cfg.MoveToCmds()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, cmd := range cmds {
|
||||
for _, btn := range AvailableMoveToBtns {
|
||||
if btn.Cmd.Name == cmd {
|
||||
btn.Cmd.Params = []string{msgHash}
|
||||
moveToBtns = append(moveToBtns, btn)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return moveToBtns
|
||||
}
|
||||
+4839
File diff suppressed because it is too large
Load Diff
+396
@@ -0,0 +1,396 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
"github.com/zakirullin/files.md/server/pkg/txt"
|
||||
)
|
||||
|
||||
var (
|
||||
Now = time.Now
|
||||
mu sync.Mutex
|
||||
userLocks map[string]*sync.Mutex
|
||||
|
||||
inboxMarkerPrefix = regexp.MustCompile(`^- \[[ xX]\] `)
|
||||
inboxHeaderRegex = regexp.MustCompile(`^#### `)
|
||||
// inboxEntryPrefix matches the prefix of an inbox entry: `- [ ] ` /
|
||||
// `- [x] ` marker followed by an optional `HH:MM` timestamp.
|
||||
inboxEntryPrefix = regexp.MustCompile(`^- \[[ xX]\] (?:` + "`" + `\d{2}:\d{2}` + "` )?")
|
||||
)
|
||||
|
||||
// stripInboxEntryPrefix removes the optional `- [ ]` / `- [x] ` task marker
|
||||
// and the “ `HH:MM` “ timestamp so only the entry body remains.
|
||||
func stripInboxEntryPrefix(block string) string {
|
||||
return inboxEntryPrefix.ReplaceAllString(block, "")
|
||||
}
|
||||
|
||||
// chatBlockHash returns a stable identifier for an inbox block. We hash only
|
||||
// the timestamped first line (after stripping the `- [ ]`/`- [x] ` marker)
|
||||
// so the identifier survives two mutations the bot makes to a block:
|
||||
// - completion toggle `- [ ]` ↔ `- [x]`
|
||||
// - forward-collapse appending continuation lines to the first block
|
||||
// (see saveFromTextMsg → createOrAdd). A keyboard built for message 1
|
||||
// must still resolve after message 2 is collapsed into the same block.
|
||||
//
|
||||
// Collision scope: two separate entries with the same “ `HH:MM` “ timestamp
|
||||
// AND identical first line would hash the same. Per-minute resolution makes
|
||||
// this rare in practice, and the outcome (acting on the older entry) is
|
||||
// harmless — it's still the user's own content with the same first line.
|
||||
// !!! TIME IS INCLUDED in the hash !!!
|
||||
func chatBlockHash(block string) string {
|
||||
stripped := inboxMarkerPrefix.ReplaceAllString(block, "")
|
||||
firstLine := strings.SplitN(stripped, "\n", 2)[0]
|
||||
return fs.Hash(firstLine)
|
||||
}
|
||||
|
||||
// findChatMsgByHash returns (blockIndex, block, true) for the first
|
||||
// non-header block whose hash matches msgHash. Returns (-1, "", false) if no
|
||||
// match is found.
|
||||
func findChatMsgByHash(content, msgHash string) (int, string, bool) {
|
||||
blocks := readChatMsgs(content)
|
||||
for i, block := range blocks {
|
||||
if inboxHeaderRegex.MatchString(block) {
|
||||
continue
|
||||
}
|
||||
if chatBlockHash(block) == msgHash {
|
||||
return i, block, true
|
||||
}
|
||||
}
|
||||
return -1, "", false
|
||||
}
|
||||
|
||||
// renameChatMsg replaces the body of the block identified by msgHash with
|
||||
// newBody, preserving the `- [ ] `/`- [x] ` marker and the “ `HH:MM` “
|
||||
// timestamp. Returns the rewritten file content.
|
||||
func renameChatMsg(content, msgHash, newBody string) (string, error) {
|
||||
blocks := readChatMsgs(content)
|
||||
idx := -1
|
||||
for i, block := range blocks {
|
||||
if inboxHeaderRegex.MatchString(block) {
|
||||
continue
|
||||
}
|
||||
if chatBlockHash(block) == msgHash {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx == -1 {
|
||||
return "", fmt.Errorf("inbox block not found for hash %q", msgHash)
|
||||
}
|
||||
|
||||
prefix := inboxEntryPrefix.FindString(blocks[idx])
|
||||
newBody = strings.TrimSpace(strings.ReplaceAll(newBody, "\n", " "))
|
||||
blocks[idx] = prefix + newBody
|
||||
|
||||
return strings.Join(blocks, "\n"), nil
|
||||
}
|
||||
|
||||
// appendToChatMsg appends newText to the chat block identified by msgHash,
|
||||
// preserving the marker/timestamp prefix. The appended text becomes a new
|
||||
// indented continuation line under the original entry.
|
||||
func appendToChatMsg(content, msgHash, newText string) (string, error) {
|
||||
blocks := readChatMsgs(content)
|
||||
idx := -1
|
||||
for i, block := range blocks {
|
||||
if inboxHeaderRegex.MatchString(block) {
|
||||
continue
|
||||
}
|
||||
if chatBlockHash(block) == msgHash {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx == -1 {
|
||||
return "", fmt.Errorf("chat block not found for hash %q", msgHash)
|
||||
}
|
||||
|
||||
newText = strings.TrimRight(newText, "\n")
|
||||
if newText == "" {
|
||||
return content, nil
|
||||
}
|
||||
blocks[idx] = strings.TrimRight(blocks[idx], "\n") + "\n" + newText
|
||||
|
||||
return strings.Join(blocks, "\n"), nil
|
||||
}
|
||||
|
||||
// appendToChat writes a new entry to Inbox.md and returns its stable hash.
|
||||
func (b *Bot) appendToChat(content string, timezone *time.Location) (string, error) {
|
||||
exists, err := b.fs.Exists(fs.DirUserRoot, fs.ChatFilename)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("appendToInbox: %w", err)
|
||||
}
|
||||
|
||||
content = strings.TrimSpace(content)
|
||||
|
||||
var md string
|
||||
if exists {
|
||||
md, err = b.fs.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("appendToInbox: %w", err)
|
||||
}
|
||||
md = txt.NormNewLines(md)
|
||||
md = strings.TrimSpace(md)
|
||||
if len(md) != 0 {
|
||||
md += "\n"
|
||||
}
|
||||
}
|
||||
|
||||
// Add today's header if it doesn't exist
|
||||
if !strings.Contains(md, todayHeader(timezone)) {
|
||||
md += todayHeader(timezone) + "\n"
|
||||
}
|
||||
|
||||
// Format timestamp with timezone
|
||||
// TODO should we use timezone here?
|
||||
timestamp := now().In(timezone).Format("`15:04`")
|
||||
|
||||
newEntry := fmt.Sprintf("- [ ] %s %s", timestamp, content)
|
||||
md += newEntry + "\n"
|
||||
|
||||
if err := b.fs.Write(fs.DirUserRoot, fs.ChatFilename, md); err != nil {
|
||||
return "", fmt.Errorf("appendToInbox: %w", err)
|
||||
}
|
||||
|
||||
return chatBlockHash(newEntry), nil
|
||||
}
|
||||
|
||||
// moveFromChat passes the messages identified by msgHashes to the callback.
|
||||
// On callback success, it removes those messages from the chat file.
|
||||
// A msgHash is the stable hash returned by chatBlockHash; it survives the
|
||||
// `[ ]` ↔ `[x]` completion toggle.
|
||||
// On collapse=false the callback is called once per message.
|
||||
func (b *Bot) moveFromChat(
|
||||
callback func(content string, timestamp time.Time) error,
|
||||
collapse bool,
|
||||
msgHashes ...string,
|
||||
) error {
|
||||
key, err := b.fs.SafePath(fs.DirUserRoot, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get safe path: %w", err)
|
||||
}
|
||||
|
||||
lock := userLock(key)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
content, err := b.fs.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blocks := readChatMsgs(content)
|
||||
|
||||
// Build hash -> block-index for every non-header block. Validate that all
|
||||
// requested hashes resolve to real blocks.
|
||||
hashToBlockIndex := make(map[string]int)
|
||||
hasAnyMsg := false
|
||||
for i, block := range blocks {
|
||||
if inboxHeaderRegex.MatchString(block) {
|
||||
continue
|
||||
}
|
||||
hasAnyMsg = true
|
||||
hashToBlockIndex[chatBlockHash(block)] = i
|
||||
}
|
||||
if !hasAnyMsg {
|
||||
return fmt.Errorf("no messages found")
|
||||
}
|
||||
resolvedBlockIndices := make([]int, 0, len(msgHashes))
|
||||
for _, h := range msgHashes {
|
||||
idx, ok := hashToBlockIndex[h]
|
||||
if !ok {
|
||||
// Recurring-schedule callback_data is tight on Telegram's 64-byte
|
||||
// limit so the picker truncates msgHash to its prefix. Fall back
|
||||
// to prefix match here.
|
||||
for full, i := range hashToBlockIndex {
|
||||
if strings.HasPrefix(full, h) {
|
||||
idx = i
|
||||
ok = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("msgHash %q not found in inbox", h)
|
||||
}
|
||||
resolvedBlockIndices = append(resolvedBlockIndices, idx)
|
||||
}
|
||||
|
||||
// Process in ascending block-index order so removal later is deterministic.
|
||||
sort.Ints(resolvedBlockIndices)
|
||||
|
||||
// Collect specified messages from inbox.
|
||||
var msgs []struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
index int
|
||||
}
|
||||
for _, blockIndex := range resolvedBlockIndices {
|
||||
block := blocks[blockIndex]
|
||||
|
||||
// Find closest header above target msg for date context
|
||||
var headerDate string
|
||||
for i := blockIndex - 1; i >= 0; i-- {
|
||||
if inboxHeaderRegex.MatchString(blocks[i]) {
|
||||
headerDate = blocks[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Strip optional `- [ ] ` / `- [x] ` marker, then optional `HH:MM`
|
||||
// timestamp. A plain checklist line without timestamp is treated as a
|
||||
// 00:00 entry on the header date.
|
||||
recordContent := inboxMarkerPrefix.ReplaceAllString(block, "")
|
||||
timeStr := "00:00"
|
||||
tsMatch := regexp.MustCompile("^`(\\d{2}:\\d{2})` ").FindStringSubmatch(recordContent)
|
||||
if tsMatch != nil {
|
||||
timeStr = tsMatch[1]
|
||||
recordContent = recordContent[len(tsMatch[0]):]
|
||||
}
|
||||
|
||||
// Parse full timestamp from header date + time. Fall back to today
|
||||
// when the entry has no `#### date` header above it (plain `- [ ] body`).
|
||||
var timestamp time.Time
|
||||
dateRegex := regexp.MustCompile(`^#### (\d{1,2}) ([A-Za-z]+), [A-Za-z]+`)
|
||||
dateMatches := dateRegex.FindStringSubmatch(headerDate)
|
||||
if len(dateMatches) >= 3 {
|
||||
dateTimeStr := fmt.Sprintf("%s %s %s", dateMatches[1], dateMatches[2], timeStr)
|
||||
parsed, err := time.Parse("2 January 15:04", dateTimeStr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse timestamp for block %d: %w", blockIndex, err)
|
||||
}
|
||||
timestamp = parsed
|
||||
} else {
|
||||
today := now()
|
||||
t, err := time.Parse("15:04", timeStr)
|
||||
if err == nil {
|
||||
timestamp = time.Date(today.Year(), today.Month(), today.Day(), t.Hour(), t.Minute(), 0, 0, today.Location())
|
||||
} else {
|
||||
timestamp = today
|
||||
}
|
||||
}
|
||||
|
||||
msgs = append(msgs, struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
index int
|
||||
}{
|
||||
content: recordContent,
|
||||
timestamp: timestamp,
|
||||
index: blockIndex,
|
||||
})
|
||||
}
|
||||
|
||||
// First we save all the messages to files, only then we remove them from the inbox.
|
||||
if collapse {
|
||||
content := strings.Builder{}
|
||||
for _, msg := range msgs {
|
||||
content.WriteString(msg.content)
|
||||
content.WriteString("\n")
|
||||
}
|
||||
err = callback(strings.TrimSpace(content.String()), msgs[0].timestamp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("callback failed: %w", err)
|
||||
}
|
||||
} else {
|
||||
for _, msg := range msgs {
|
||||
if err := callback(msg.content, msg.timestamp); err != nil {
|
||||
return fmt.Errorf("callback failed: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
blocksToRemove := make(map[int]bool)
|
||||
for _, msg := range msgs {
|
||||
blocksToRemove[msg.index] = true
|
||||
}
|
||||
newBlocks := make([]string, 0)
|
||||
for i, block := range blocks {
|
||||
if blocksToRemove[i] {
|
||||
continue
|
||||
}
|
||||
newBlocks = append(newBlocks, block)
|
||||
}
|
||||
modifiedContent := strings.TrimSpace(strings.Join(newBlocks, "\n"))
|
||||
|
||||
return b.fs.Write(fs.DirUserRoot, fs.ChatFilename, modifiedContent)
|
||||
}
|
||||
|
||||
// readChatMsgs parses content into logical blocks
|
||||
// Returns slice where each element is either a header or a complete record
|
||||
func readChatMsgs(content string) []string {
|
||||
content = txt.NormNewLines(content)
|
||||
lines := strings.Split(content, "\n")
|
||||
|
||||
headerRegex := regexp.MustCompile(`^#### `)
|
||||
// Block start: a `- [ ] ` / `- [x] ` checklist line (timestamp optional).
|
||||
timestampRegex := regexp.MustCompile(`^- \[[ xX]\] `)
|
||||
|
||||
var blocks []string
|
||||
var currentBlock strings.Builder
|
||||
|
||||
for _, line := range lines {
|
||||
isHeader := headerRegex.MatchString(line)
|
||||
isTimestamp := timestampRegex.MatchString(line)
|
||||
|
||||
if isHeader {
|
||||
// Save previous block if exists
|
||||
if currentBlock.Len() > 0 {
|
||||
blocks = append(blocks, strings.TrimSpace(currentBlock.String()))
|
||||
currentBlock.Reset()
|
||||
}
|
||||
// DisplayName is always its own block
|
||||
blocks = append(blocks, line)
|
||||
} else if isTimestamp {
|
||||
// Save previous block if exists
|
||||
if currentBlock.Len() > 0 {
|
||||
blocks = append(blocks, strings.TrimSpace(currentBlock.String()))
|
||||
currentBlock.Reset()
|
||||
}
|
||||
// Start new block with timestamp
|
||||
currentBlock.WriteString(line)
|
||||
} else {
|
||||
// Continue current block or start new block
|
||||
if currentBlock.Len() > 0 {
|
||||
currentBlock.WriteString("\n")
|
||||
currentBlock.WriteString(line)
|
||||
} else {
|
||||
currentBlock.WriteString(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add final block
|
||||
if currentBlock.Len() > 0 {
|
||||
blocks = append(blocks, strings.TrimSpace(currentBlock.String()))
|
||||
}
|
||||
|
||||
return blocks
|
||||
}
|
||||
|
||||
func todayHeader(timezone *time.Location) string {
|
||||
nowTZ := now().In(timezone)
|
||||
return fmt.Sprintf("#### %d %s, %s", nowTZ.Day(), nowTZ.Format("January"), nowTZ.Weekday())
|
||||
}
|
||||
|
||||
func userLock(rootPath string) *sync.Mutex {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
if userLocks == nil {
|
||||
userLocks = make(map[string]*sync.Mutex)
|
||||
}
|
||||
if lock, exists := userLocks[rootPath]; exists {
|
||||
return lock
|
||||
}
|
||||
|
||||
newLock := &sync.Mutex{}
|
||||
userLocks[rootPath] = newLock
|
||||
|
||||
return newLock
|
||||
}
|
||||
@@ -0,0 +1,755 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zakirullin/files.md/server/db"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
"github.com/zakirullin/files.md/server/pkg/tg"
|
||||
)
|
||||
|
||||
func TestReadMessagesEmpty(t *testing.T) {
|
||||
r := require.New(t)
|
||||
result := readChatMsgs("")
|
||||
r.Empty(result)
|
||||
}
|
||||
|
||||
func TestReadMessagesOnlyHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
result := readChatMsgs("#### 27 June, Friday")
|
||||
r.Equal([]string{"#### 27 June, Friday"}, result)
|
||||
}
|
||||
|
||||
func TestReadMessagesSingleRecord(t *testing.T) {
|
||||
r := require.New(t)
|
||||
result := readChatMsgs("- [ ] `01:01` Simple record")
|
||||
r.Equal([]string{"- [ ] `01:01` Simple record"}, result)
|
||||
}
|
||||
|
||||
func TestReadMessagesHeaderWithRecord(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := "#### 27 June, Friday\n- [ ] `01:01` Simple record"
|
||||
result := readChatMsgs(content)
|
||||
r.Equal([]string{"#### 27 June, Friday", "- [ ] `01:01` Simple record"}, result)
|
||||
}
|
||||
|
||||
func TestReadMessagesMultilineRecord(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := "#### 27 June, Friday\n- [ ] `01:01` Multiline\nc\nontent"
|
||||
result := readChatMsgs(content)
|
||||
r.Equal([]string{"#### 27 June, Friday", "- [ ] `01:01` Multiline\nc\nontent"}, result)
|
||||
}
|
||||
|
||||
func TestReadMessagesMultipleRecords(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := "#### 27 June, Friday\n- [ ] `01:01` First record\n- [ ] `02:02` Second record"
|
||||
result := readChatMsgs(content)
|
||||
r.Equal([]string{"#### 27 June, Friday", "- [ ] `01:01` First record", "- [ ] `02:02` Second record"}, result)
|
||||
}
|
||||
|
||||
func TestReadMessagesMultipleHeaders(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := "#### 27 June, Friday\n- [ ] `01:01` First day\n#### 28 June, Saturday\n- [ ] `02:02` Second day"
|
||||
result := readChatMsgs(content)
|
||||
r.Equal([]string{"#### 27 June, Friday", "- [ ] `01:01` First day", "#### 28 June, Saturday", "- [ ] `02:02` Second day"}, result)
|
||||
}
|
||||
|
||||
func TestReadMessagesWindowsLineEndings(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := "#### 27 June, Friday\r\n- [ ] `01:01` Windows record"
|
||||
result := readChatMsgs(content)
|
||||
r.Equal([]string{"#### 27 June, Friday", "- [ ] `01:01` Windows record"}, result)
|
||||
}
|
||||
|
||||
func TestReadMessagesWithEmptyLines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := "#### 27 June, Friday\n\n- [ ] `01:01` Record with\n\nempty lines"
|
||||
result := readChatMsgs(content)
|
||||
r.Equal([]string{"#### 27 June, Friday", "- [ ] `01:01` Record with\n\nempty lines"}, result)
|
||||
}
|
||||
|
||||
func TestReadMessagesInvalidTimestamp(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := "#### 27 June, Friday\n`not timestamp` Should be continuation\n- [ ] `01:01` Real record"
|
||||
result := readChatMsgs(content)
|
||||
r.Equal([]string{"#### 27 June, Friday", "`not timestamp` Should be continuation", "- [ ] `01:01` Real record"}, result)
|
||||
}
|
||||
|
||||
func TestSaveToChatNewFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
savedNow := now
|
||||
defer func() { now = savedNow }()
|
||||
now = func() time.Time {
|
||||
return time.Date(2024, 6, 27, 1, 1, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
h, err := bot.appendToChat("Test content", time.UTC)
|
||||
r.NoError(err)
|
||||
r.Equal(chatBlockHash("- [ ] `01:01` Test content"), h)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
r.Equal("#### 27 June, Thursday\n- [ ] `01:01` Test content\n", content)
|
||||
}
|
||||
|
||||
func TestSaveToChatExistingFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
savedNow := now
|
||||
defer func() { now = savedNow }()
|
||||
now = func() time.Time {
|
||||
return time.Date(2024, 6, 27, 1, 1, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
err = userFS.Write(fs.DirUserRoot, fs.ChatFilename, "#### 27 June, Thursday\n`00:30` Existing content\n")
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
h, err := bot.appendToChat("New content", time.UTC)
|
||||
r.NoError(err)
|
||||
r.Equal(chatBlockHash("- [ ] `01:01` New content"), h)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
r.Equal("#### 27 June, Thursday\n`00:30` Existing content\n- [ ] `01:01` New content\n", content)
|
||||
}
|
||||
|
||||
func TestSaveToChatNewDay(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
savedNow := now
|
||||
defer func() { now = savedNow }()
|
||||
now = func() time.Time {
|
||||
return time.Date(2024, 6, 28, 1, 1, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
err = userFS.Write(fs.DirUserRoot, fs.ChatFilename, "#### 27 June, Thursday\n`00:30` Yesterday content\n")
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
h, err := bot.appendToChat("Today content", time.UTC)
|
||||
r.NoError(err)
|
||||
r.Equal(chatBlockHash("- [ ] `01:01` Today content"), h)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
r.Equal("#### 27 June, Thursday\n`00:30` Yesterday content\n#### 28 June, Friday\n- [ ] `01:01` Today content\n", content)
|
||||
}
|
||||
|
||||
func TestSaveToChatWithImage(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
savedNow := now
|
||||
defer func() { now = savedNow }()
|
||||
now = func() time.Time {
|
||||
return time.Date(2024, 6, 27, 1, 1, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
h, err := bot.appendToChat(" Image description", time.UTC)
|
||||
r.NoError(err)
|
||||
r.Equal(chatBlockHash("- [ ] `01:01`  Image description"), h)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
r.Equal("#### 27 June, Thursday\n- [ ] `01:01`  Image description\n", content)
|
||||
}
|
||||
|
||||
func TestSaveToChatEmptyFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
savedNow := now
|
||||
defer func() { now = savedNow }()
|
||||
now = func() time.Time {
|
||||
return time.Date(2024, 6, 27, 1, 1, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
err = userFS.Write(fs.DirUserRoot, fs.ChatFilename, "")
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
h, err := bot.appendToChat("Test content", time.UTC)
|
||||
r.NoError(err)
|
||||
r.Equal(chatBlockHash("- [ ] `01:01` Test content"), h)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
r.Equal("#### 27 June, Thursday\n- [ ] `01:01` Test content\n", content)
|
||||
}
|
||||
|
||||
//func TestSaveToChatWithTimezone(t *testing.T) {
|
||||
// r := require.New(t)
|
||||
//
|
||||
// savedNow := now
|
||||
// defer func() { now = savedNow }()
|
||||
// now = func() time.Time {
|
||||
// return time.Date(2024, 6, 27, 1, 1, 0, 0, time.UTC)
|
||||
// }
|
||||
//
|
||||
// userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
// r.NoError(err)
|
||||
//
|
||||
// bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
//
|
||||
// // Use EST timezone (UTC-5)
|
||||
// est, err := time.LoadLocation("America/New_York")
|
||||
// r.NoError(err)
|
||||
//
|
||||
// index, err := bot.saveToChat("Test content", est)
|
||||
// r.NoError(err)
|
||||
// r.Equal(1, index)
|
||||
//
|
||||
// content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
// r.NoError(err)
|
||||
// // Should use EST time which is 20:01 (8:01 PM) the previous day
|
||||
// r.Contains(content, "`20:01` Test content")
|
||||
//}
|
||||
|
||||
// Test normal case - properly formatted content
|
||||
func TestReadMessages_NormalCase(t *testing.T) {
|
||||
content := `#### 1 July, Tuesday
|
||||
- [ ] ` + "`15:19`" + ` Пройтись на улице
|
||||
#### 2 July, Wednesday
|
||||
- [ ] ` + "`10:30`" + ` Почитать книгу`
|
||||
|
||||
result := readChatMsgs(content)
|
||||
expected := []string{
|
||||
"#### 1 July, Tuesday",
|
||||
"- [ ] `15:19` Пройтись на улице",
|
||||
"#### 2 July, Wednesday",
|
||||
"- [ ] `10:30` Почитать книгу",
|
||||
}
|
||||
|
||||
if len(result) != len(expected) {
|
||||
t.Fatalf("Expected %d blocks, got %d", len(expected), len(result))
|
||||
}
|
||||
|
||||
for i, block := range result {
|
||||
if block != expected[i] {
|
||||
t.Errorf("Block %d mismatch:\nExpected: %q\nGot: %q", i, expected[i], block)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test message without timestamp - this could be the source of the issue
|
||||
func TestReadMessages_MessageWithoutTimestamp(t *testing.T) {
|
||||
content := `#### 1 July, Tuesday
|
||||
- [ ] ` + "`15:19`" + ` Пройтись на улице
|
||||
Провести звонок с Нео
|
||||
#### 2 July, Wednesday
|
||||
- [ ] ` + "`10:30`" + ` Почитать книгу`
|
||||
|
||||
result := readChatMsgs(content)
|
||||
|
||||
// The message without timestamp should be grouped with the previous timestamped message
|
||||
expected := []string{
|
||||
"#### 1 July, Tuesday",
|
||||
"- [ ] `15:19` Пройтись на улице\nПровести звонок с Нео",
|
||||
"#### 2 July, Wednesday",
|
||||
"- [ ] `10:30` Почитать книгу",
|
||||
}
|
||||
|
||||
if len(result) != len(expected) {
|
||||
t.Fatalf("Expected %d blocks, got %d\nGot: %v", len(expected), len(result), result)
|
||||
}
|
||||
|
||||
for i, block := range result {
|
||||
if block != expected[i] {
|
||||
t.Errorf("Block %d mismatch:\nExpected: %q\nGot: %q", i, expected[i], block)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//func TestReadMessages_MultipleMessagesWithoutTimestamps(t *testing.T) {
|
||||
// content := `#### 1 July, Tuesday
|
||||
//` + "`15:19`" + ` Пройтись на улице
|
||||
//Провести звонок с Нео
|
||||
//Купить молоко
|
||||
//#### 2 July, Wednesday
|
||||
//Почитать книгу
|
||||
//` + "`10:30`" + ` Сходить в магазин`
|
||||
//
|
||||
// result := readMessages(content)
|
||||
//
|
||||
// // Check what happens with multiple untimestamped messages
|
||||
// t.Logf("Result blocks: %v", result)
|
||||
//
|
||||
// // This should reveal how readMessages handles content without timestamps
|
||||
// headerRegex := regexp.MustCompile(`^#### `)
|
||||
// timestampRegex := regexp.MustCompile(`^` + "`" + `\d{2}:\d{2}` + "`" + ` `)
|
||||
//
|
||||
// for i, block := range result {
|
||||
// isHeader := headerRegex.MatchString(block)
|
||||
// hasTimestamp := timestampRegex.MatchString(block)
|
||||
// t.Logf("Block %d: isHeader=%v, hasTimestamp=%v, content=%q", i, isHeader, hasTimestamp, block)
|
||||
//
|
||||
// if !isHeader && !hasTimestamp {
|
||||
// t.Errorf("Found record without timestamp: %q", block)
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
//func TestReadMessages_ContentAfterHeaderWithoutTimestamp(t *testing.T) {
|
||||
// content := `#### 1 July, Tuesday
|
||||
//#### 2 July, Wednesday
|
||||
//Почитать книгу
|
||||
//` + "`10:30`" + ` Сходить в магазин`
|
||||
//
|
||||
// result := readMessages(content)
|
||||
//
|
||||
// t.Logf("Result blocks: %v", result)
|
||||
//
|
||||
// // This case might be critical - content right after header without timestamp
|
||||
// headerRegex := regexp.MustCompile(`^#### `)
|
||||
// timestampRegex := regexp.MustCompile(`^` + "`" + `\d{2}:\d{2}` + "`" + ` `)
|
||||
//
|
||||
// for i, block := range result {
|
||||
// isHeader := headerRegex.MatchString(block)
|
||||
// hasTimestamp := timestampRegex.MatchString(block)
|
||||
// t.Logf("Block %d: isHeader=%v, hasTimestamp=%v, content=%q", i, isHeader, hasTimestamp, block)
|
||||
//
|
||||
// if !isHeader && !hasTimestamp {
|
||||
// t.Errorf("Found record without timestamp: %q", block)
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
// Test multiline message formatting
|
||||
func TestReadBlocks_MultilineMessage(t *testing.T) {
|
||||
content := `#### 1 July, Tuesday
|
||||
- [ ] ` + "`15:19`" + ` Пройтись на улице
|
||||
и купить хлеб
|
||||
в магазине
|
||||
#### 2 July, Wednesday
|
||||
- [ ] ` + "`10:30`" + ` Почитать книгу`
|
||||
|
||||
result := readChatMsgs(content)
|
||||
expected := []string{
|
||||
"#### 1 July, Tuesday",
|
||||
"- [ ] `15:19` Пройтись на улице\nи купить хлеб\nв магазине",
|
||||
"#### 2 July, Wednesday",
|
||||
"- [ ] `10:30` Почитать книгу",
|
||||
}
|
||||
|
||||
if len(result) != len(expected) {
|
||||
t.Fatalf("Expected %d blocks, got %d", len(expected), len(result))
|
||||
}
|
||||
|
||||
for i, block := range result {
|
||||
if block != expected[i] {
|
||||
t.Errorf("Block %d mismatch:\nExpected: %q\nGot: %q", i, expected[i], block)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBlocksHasTimestamp(t *testing.T) {
|
||||
content := `#### 1 July, Tuesday
|
||||
- [ ] ` + "`15:19`" + ` Do some stuff
|
||||
Arrange a call with Neo
|
||||
#### 2 July, Wednesday
|
||||
- [ ] ` + "`10:30`" + ` Read a book`
|
||||
|
||||
messages := readChatMsgs(content)
|
||||
|
||||
// Filter to find record messages (not headers)
|
||||
headerRegex := regexp.MustCompile(`^#### `)
|
||||
var recordIndices []int
|
||||
var records []string
|
||||
|
||||
for i, message := range messages {
|
||||
if !headerRegex.MatchString(message) {
|
||||
recordIndices = append(recordIndices, i)
|
||||
records = append(records, message)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Found %d records:", len(records))
|
||||
for i, record := range records {
|
||||
t.Logf("Record %d: %q", i, record)
|
||||
}
|
||||
|
||||
timestampRegex := regexp.MustCompile(`^- \[[ xX]\] ` + "`" + `\d{2}:\d{2}` + "`" + ` `)
|
||||
for i, record := range records {
|
||||
hasTimestamp := timestampRegex.MatchString(record)
|
||||
t.Logf("Record %d has timestamp: %v", i, hasTimestamp)
|
||||
if !hasTimestamp {
|
||||
t.Errorf("Record %d missing timestamp: %q", i, record)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//func TestReadMessages_ExactIssueScenario(t *testing.T) {
|
||||
// content := `#### 1 July, Tuesday
|
||||
//#### 2 July, Wednesday
|
||||
//Почитать книгу
|
||||
//` + "`15:19`" + ` Пройтись на улице
|
||||
//Провести звонок с Нео`
|
||||
//
|
||||
// result := readMessages(content)
|
||||
//
|
||||
// t.Logf("Input content:")
|
||||
// t.Logf("%q", content)
|
||||
// t.Logf("Parsed blocks:")
|
||||
// for i, block := range result {
|
||||
// t.Logf("Block %d: %q", i, block)
|
||||
// }
|
||||
//
|
||||
// // Check for records without timestamps
|
||||
// headerRegex := regexp.MustCompile(`^#### `)
|
||||
// timestampRegex := regexp.MustCompile(`^` + "`" + `\d{2}:\d{2}` + "`" + ` `)
|
||||
//
|
||||
// for i, block := range result {
|
||||
// isHeader := headerRegex.MatchString(block)
|
||||
// hasTimestamp := timestampRegex.MatchString(block)
|
||||
//
|
||||
// if !isHeader && !hasTimestamp {
|
||||
// t.Errorf("Found record without timestamp at index %d: %q", i, block)
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
// Test timestamp pattern matching
|
||||
func TestTimestampPatternMatching(t *testing.T) {
|
||||
timestampRegex := regexp.MustCompile(`^` + "`" + `\d{2}:\d{2}` + "`" + ` `)
|
||||
|
||||
validTimestamps := []string{
|
||||
"`15:19` Пройтись на улице",
|
||||
"`10:30` Почитать книгу",
|
||||
"`23:59` Test message",
|
||||
}
|
||||
|
||||
invalidTimestamps := []string{
|
||||
"15:19 Пройтись на улице", // No backticks
|
||||
"Пройтись на улице", // No timestamp
|
||||
"15:19` Пройтись на улице", // Missing opening backtick
|
||||
"`15:19 Пройтись на улице", // Missing closing backtick
|
||||
"`5:19` Пройтись на улице", // Single digit hour
|
||||
"`15:9` Пройтись на улице", // Single digit minute
|
||||
}
|
||||
|
||||
for _, ts := range validTimestamps {
|
||||
if !timestampRegex.MatchString(ts) {
|
||||
t.Errorf("Valid timestamp not matched: %q", ts)
|
||||
}
|
||||
}
|
||||
|
||||
for _, ts := range invalidTimestamps {
|
||||
if timestampRegex.MatchString(ts) {
|
||||
t.Errorf("Invalid timestamp matched: %q", ts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test what happens when saveToChat adds content without proper formatting
|
||||
func TestSaveToChat_ContentAddition(t *testing.T) {
|
||||
// Mock the current time for testing
|
||||
originalNow := Now
|
||||
defer func() { Now = originalNow }()
|
||||
|
||||
Now = func() time.Time {
|
||||
return time.Date(2024, 7, 2, 15, 19, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
timezone := time.UTC
|
||||
|
||||
// Test content formatting in saveToChat
|
||||
content := "Arrange call with Neo"
|
||||
timestamp := Now().In(timezone).Format("`15:04`")
|
||||
expectedFormat := timestamp + " " + content + "\n"
|
||||
|
||||
t.Logf("Content: %q", content)
|
||||
t.Logf("Timestamp: %q", timestamp)
|
||||
t.Logf("Expected format: %q", expectedFormat)
|
||||
|
||||
// This should match the format from saveToChat function
|
||||
if !strings.Contains(expectedFormat, "`15:19`") {
|
||||
t.Errorf("Timestamp format is incorrect: %q", expectedFormat)
|
||||
}
|
||||
}
|
||||
|
||||
// Test edge case: empty content handling
|
||||
func TestReadMessages_EmptyContent(t *testing.T) {
|
||||
content := ""
|
||||
result := readChatMsgs(content)
|
||||
|
||||
if len(result) != 0 {
|
||||
t.Errorf("Expected 0 blocks for empty content, got %d: %v", len(result), result)
|
||||
}
|
||||
}
|
||||
|
||||
// Test edge case: only headers
|
||||
func TestReadMessages_OnlyHeaders(t *testing.T) {
|
||||
content := `#### 1 July, Tuesday
|
||||
#### 2 July, Wednesday`
|
||||
|
||||
result := readChatMsgs(content)
|
||||
expected := []string{
|
||||
"#### 1 July, Tuesday",
|
||||
"#### 2 July, Wednesday",
|
||||
}
|
||||
|
||||
if len(result) != len(expected) {
|
||||
t.Fatalf("Expected %d blocks, got %d", len(expected), len(result))
|
||||
}
|
||||
|
||||
for i, block := range result {
|
||||
if block != expected[i] {
|
||||
t.Errorf("Block %d mismatch:\nExpected: %q\nGot: %q", i, expected[i], block)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveFromInboxSingleRecord(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
initialContent := `#### 27 June, Thursday
|
||||
- [ ] ` + "`01:01`" + ` First record
|
||||
- [ ] ` + "`02:02`" + ` Second record
|
||||
#### 28 June, Friday
|
||||
- [ ] ` + "`03:03`" + ` Third record`
|
||||
|
||||
err = userFS.Write(fs.DirUserRoot, fs.ChatFilename, initialContent)
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
var callbackCalls []struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
callback := func(content string, timestamp time.Time) error {
|
||||
callbackCalls = append(callbackCalls, struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
}{content, timestamp})
|
||||
return nil
|
||||
}
|
||||
|
||||
err = bot.moveFromChat(callback, false, chatBlockHash("- [ ] `02:02` Second record"))
|
||||
r.NoError(err)
|
||||
|
||||
r.Len(callbackCalls, 1)
|
||||
r.Equal("Second record", callbackCalls[0].content)
|
||||
|
||||
expectedTime, _ := time.Parse("2 January 15:04", "27 June 02:02")
|
||||
r.Equal(expectedTime, callbackCalls[0].timestamp)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
|
||||
expectedContent := `#### 27 June, Thursday
|
||||
- [ ] ` + "`01:01`" + ` First record
|
||||
#### 28 June, Friday
|
||||
- [ ] ` + "`03:03`" + ` Third record`
|
||||
|
||||
r.Equal(expectedContent, content)
|
||||
}
|
||||
|
||||
func TestMoveFromInboxMultipleRecords(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
initialContent := `#### 1 July, Monday
|
||||
- [ ] ` + "`10:30`" + ` Buy groceries
|
||||
milk, bread, eggs
|
||||
- [ ] ` + "`11:45`" + ` Call mom
|
||||
#### 2 July, Tuesday
|
||||
- [ ] ` + "`09:15`" + ` Morning workout
|
||||
- [ ] ` + "`14:20`" + ` Team meeting
|
||||
discuss project timeline
|
||||
and resource allocation`
|
||||
|
||||
err = userFS.Write(fs.DirUserRoot, fs.ChatFilename, initialContent)
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
var callbackCalls []struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
callback := func(content string, timestamp time.Time) error {
|
||||
callbackCalls = append(callbackCalls, struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
}{content, timestamp})
|
||||
return nil
|
||||
}
|
||||
|
||||
err = bot.moveFromChat(callback, false,
|
||||
chatBlockHash("- [ ] `10:30` Buy groceries\nmilk, bread, eggs"),
|
||||
chatBlockHash("- [ ] `09:15` Morning workout"),
|
||||
)
|
||||
r.NoError(err)
|
||||
|
||||
r.Len(callbackCalls, 2)
|
||||
|
||||
r.Equal("Buy groceries\nmilk, bread, eggs", callbackCalls[0].content)
|
||||
expectedTime1, _ := time.Parse("2 January 15:04", "1 July 10:30")
|
||||
r.Equal(expectedTime1, callbackCalls[0].timestamp)
|
||||
|
||||
r.Equal("Morning workout", callbackCalls[1].content)
|
||||
expectedTime2, _ := time.Parse("2 January 15:04", "2 July 09:15")
|
||||
r.Equal(expectedTime2, callbackCalls[1].timestamp)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
|
||||
expectedContent := `#### 1 July, Monday
|
||||
- [ ] ` + "`11:45`" + ` Call mom
|
||||
#### 2 July, Tuesday
|
||||
- [ ] ` + "`14:20`" + ` Team meeting
|
||||
discuss project timeline
|
||||
and resource allocation`
|
||||
|
||||
r.Equal(expectedContent, content)
|
||||
}
|
||||
|
||||
func TestMoveFromChatCollapsedSingleRecord(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
initialContent := `#### 27 June, Thursday
|
||||
- [ ] ` + "`01:01`" + ` First record
|
||||
- [ ] ` + "`02:02`" + ` Second record
|
||||
#### 28 June, Friday
|
||||
- [ ] ` + "`03:03`" + ` Third record`
|
||||
|
||||
err = userFS.Write(fs.DirUserRoot, fs.ChatFilename, initialContent)
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
var callbackCalls []struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
callback := func(content string, timestamp time.Time) error {
|
||||
callbackCalls = append(callbackCalls, struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
}{content, timestamp})
|
||||
return nil
|
||||
}
|
||||
|
||||
err = bot.moveFromChat(callback, true, chatBlockHash("- [ ] `02:02` Second record"))
|
||||
r.NoError(err)
|
||||
|
||||
r.Len(callbackCalls, 1)
|
||||
r.Equal("Second record", callbackCalls[0].content)
|
||||
|
||||
expectedTime, _ := time.Parse("2 January 15:04", "27 June 02:02")
|
||||
r.Equal(expectedTime, callbackCalls[0].timestamp)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
|
||||
expectedContent := `#### 27 June, Thursday
|
||||
- [ ] ` + "`01:01`" + ` First record
|
||||
#### 28 June, Friday
|
||||
- [ ] ` + "`03:03`" + ` Third record`
|
||||
|
||||
r.Equal(expectedContent, content)
|
||||
}
|
||||
|
||||
func TestMoveFromChatCollapsedMultipleRecords(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
initialContent := `#### 1 July, Monday
|
||||
- [ ] ` + "`10:30`" + ` Buy groceries
|
||||
milk, bread, eggs
|
||||
- [ ] ` + "`11:45`" + ` Call mom
|
||||
#### 2 July, Tuesday
|
||||
- [ ] ` + "`09:15`" + ` Morning workout
|
||||
- [ ] ` + "`14:20`" + ` Team meeting
|
||||
discuss project timeline
|
||||
and resource allocation`
|
||||
|
||||
err = userFS.Write(fs.DirUserRoot, fs.ChatFilename, initialContent)
|
||||
r.NoError(err)
|
||||
|
||||
bot := NewBot(-1, tg.NewFakeTG(), userFS, db.NewFakeDB(), fakeConfig())
|
||||
|
||||
// Track callback calls
|
||||
var callbackCalls []struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
callback := func(content string, timestamp time.Time) error {
|
||||
callbackCalls = append(callbackCalls, struct {
|
||||
content string
|
||||
timestamp time.Time
|
||||
}{content, timestamp})
|
||||
return nil
|
||||
}
|
||||
|
||||
err = bot.moveFromChat(callback, true,
|
||||
chatBlockHash("- [ ] `10:30` Buy groceries\nmilk, bread, eggs"),
|
||||
chatBlockHash("- [ ] `09:15` Morning workout"),
|
||||
)
|
||||
r.NoError(err)
|
||||
|
||||
r.Len(callbackCalls, 1)
|
||||
|
||||
expectedCollapsedContent := `Buy groceries
|
||||
milk, bread, eggs
|
||||
Morning workout`
|
||||
r.Equal(expectedCollapsedContent, callbackCalls[0].content)
|
||||
|
||||
expectedTime, _ := time.Parse("2 January 15:04", "1 July 10:30")
|
||||
r.Equal(expectedTime, callbackCalls[0].timestamp)
|
||||
|
||||
content, err := userFS.Read(fs.DirUserRoot, fs.ChatFilename)
|
||||
r.NoError(err)
|
||||
|
||||
expectedContent := `#### 1 July, Monday
|
||||
- [ ] ` + "`11:45`" + ` Call mom
|
||||
#### 2 July, Tuesday
|
||||
- [ ] ` + "`14:20`" + ` Team meeting
|
||||
discuss project timeline
|
||||
and resource allocation`
|
||||
|
||||
r.Equal(expectedContent, content)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/kelseyhightower/envconfig"
|
||||
)
|
||||
|
||||
// APIURL / AppURL carry the full scheme+host (e.g. "https://api.files.md").
|
||||
// Hostnames are derived from them on demand via APIHost()/AppHost().
|
||||
type Config struct {
|
||||
WorkingDir string
|
||||
StorageDir string `default:"./storage" envconfig:"STORAGE_DIR"`
|
||||
BotAPIToken string `required:"false" envconfig:"BOT_API_TOKEN"`
|
||||
ConfigFilename string `default:"config.json"`
|
||||
APIURL string `default:"" envconfig:"API_URL"`
|
||||
AppURL string `default:"" envconfig:"APP_URL"`
|
||||
ServerCertDir string `default:"/tmp" envconfig:"CERT_DIR"`
|
||||
TokensDir string `default:"/tmp" envconfig:"TOKENS_DIR"`
|
||||
TokensSalt string `envconfig:"TOKENS_SALT"`
|
||||
ServerLogFile string `default:"/tmp/server.log" envconfig:"LOG_FILE"`
|
||||
StorageQuotaKB int64 `default:"1024" envconfig:"STORAGE_QUOTA_KB"` // 1MB
|
||||
UnlimitedQuotaIDs string `envconfig:"UNLIMITED_QUOTA_IDS"`
|
||||
}
|
||||
|
||||
func (c Config) APIHost() string { return hostOf(c.APIURL) }
|
||||
func (c Config) AppHost() string { return hostOf(c.AppURL) }
|
||||
|
||||
func hostOf(rawURL string) string {
|
||||
if rawURL == "" {
|
||||
return ""
|
||||
}
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return u.Host
|
||||
}
|
||||
|
||||
var ServerCfg Config
|
||||
|
||||
func LoadBotConfig() error {
|
||||
if err := envconfig.Process("", &ServerCfg); err != nil {
|
||||
return fmt.Errorf("can't load config: %w", err)
|
||||
}
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return fmt.Errorf("config can't get working directory: %w", err)
|
||||
}
|
||||
ServerCfg.WorkingDir = wd
|
||||
|
||||
if !filepath.IsAbs(ServerCfg.StorageDir) {
|
||||
ServerCfg.StorageDir = filepath.Join(wd, ServerCfg.StorageDir)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
// Package db provides an in-memory database for storing user-specific temporary data.
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/zakirullin/files.md/server/pkg/tg"
|
||||
)
|
||||
|
||||
// In-memory database
|
||||
var (
|
||||
hashOrPathByMsgID sync.Map
|
||||
inputExpectations sync.Map
|
||||
recentCommands sync.Map
|
||||
recentCommandsTargets sync.Map
|
||||
sentPhotoMsgIDs sync.Map
|
||||
)
|
||||
|
||||
// DB Do we need a type at all?
|
||||
type DB struct {
|
||||
UserID int64
|
||||
}
|
||||
|
||||
func NewDB(userID int64) *DB {
|
||||
return &DB{UserID: userID}
|
||||
}
|
||||
|
||||
// TODO add locks
|
||||
func (db *DB) LastKeyboardMsgID() (int, bool) {
|
||||
msgIDStr, err := os.ReadFile(tmpFilePath(db.UserID, "msgid"))
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
msgID, err := strconv.Atoi(string(msgIDStr))
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return msgID, true
|
||||
}
|
||||
|
||||
func (db *DB) SetLastKeyboardMsgID(ID int) {
|
||||
_ = os.WriteFile(tmpFilePath(db.UserID, "msgid"), []byte(strconv.Itoa(ID)), 0o644)
|
||||
}
|
||||
|
||||
func (db *DB) DelLastKeyboardMsgID() {
|
||||
_ = os.Remove(tmpFilePath(db.UserID, "msgid"))
|
||||
}
|
||||
|
||||
func (db *DB) InputExpectation() *tg.Cmd {
|
||||
val, ok := inputExpectations.Load(inputExpectationKey(db.UserID))
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
cmd := val.(tg.Cmd)
|
||||
return &cmd
|
||||
}
|
||||
|
||||
func (db *DB) SetInputExpectation(cmd tg.Cmd) {
|
||||
inputExpectations.Store(inputExpectationKey(db.UserID), cmd)
|
||||
}
|
||||
|
||||
func (db *DB) DelInputExpectation() {
|
||||
inputExpectations.Delete(inputExpectationKey(db.UserID))
|
||||
}
|
||||
|
||||
// HashOrPathByMsgID returns the target the bot rendered for this msgID -
|
||||
// either a chat-block hash or an absolute file path. Callers distinguish
|
||||
// the two by the leading "/" (paths) vs no slash (hash).
|
||||
func (db *DB) HashOrPathByMsgID(msgID int) (string, bool) {
|
||||
v, ok := hashOrPathByMsgID.Load(hashOrPathByMsgIDKey(db.UserID, msgID))
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return v.(string), true
|
||||
}
|
||||
|
||||
func (db *DB) SetHashOrPathByMsgID(msgID int, value string) {
|
||||
hashOrPathByMsgID.Store(hashOrPathByMsgIDKey(db.UserID, msgID), value)
|
||||
}
|
||||
|
||||
func (db *DB) RecentCommand() (string, bool) {
|
||||
cmd, ok := recentCommands.Load(db.UserID)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return cmd.(string), true
|
||||
}
|
||||
|
||||
func (db *DB) SetRecentCommand(cmd string) {
|
||||
recentCommands.Store(db.UserID, cmd)
|
||||
}
|
||||
|
||||
func (db *DB) RecentCommandParams() ([]string, bool) {
|
||||
params, ok := recentCommandsTargets.Load(db.UserID)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return params.([]string), true
|
||||
}
|
||||
|
||||
func (db *DB) SetRecentCommandParams(params []string) {
|
||||
recentCommandsTargets.Store(db.UserID, params)
|
||||
}
|
||||
|
||||
func (db *DB) AddImgMsgID(msgID int) {
|
||||
key := photoMsgIDKey(db.UserID)
|
||||
if val, ok := sentPhotoMsgIDs.Load(key); ok {
|
||||
ids := val.([]int)
|
||||
sentPhotoMsgIDs.Store(key, append(ids, msgID))
|
||||
} else {
|
||||
sentPhotoMsgIDs.Store(key, []int{msgID})
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) ImgMsgID() ([]int, bool) {
|
||||
key := photoMsgIDKey(db.UserID)
|
||||
val, ok := sentPhotoMsgIDs.Load(key)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
ids, _ := val.([]int)
|
||||
return ids, true
|
||||
}
|
||||
|
||||
func (db *DB) DelImgMsgID() {
|
||||
key := photoMsgIDKey(db.UserID)
|
||||
sentPhotoMsgIDs.Delete(key)
|
||||
}
|
||||
|
||||
func photoMsgIDKey(userID int64) string {
|
||||
return fmt.Sprintf("%d:sentPhotoMsgIDs", userID)
|
||||
}
|
||||
|
||||
func inputExpectationKey(userID int64) string {
|
||||
return fmt.Sprintf("%d:inputExpectations", userID)
|
||||
}
|
||||
|
||||
func hashOrPathByMsgIDKey(userID int64, msgID int) string {
|
||||
return fmt.Sprintf("%d:hashOrPathByMsgID:%d", userID, msgID)
|
||||
}
|
||||
|
||||
func tmpFilePath(userID int64, name string) string {
|
||||
return fmt.Sprintf("%s/%d.%s", os.TempDir(), userID, name)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"github.com/zakirullin/files.md/server/pkg/tg"
|
||||
)
|
||||
|
||||
// FakeDB is a fake database, used for testing
|
||||
// We don't have to clear it after each test
|
||||
type FakeDB struct {
|
||||
HashOrPathByMID string
|
||||
InputExpectationCMD *tg.Cmd
|
||||
LastKeyboardMID int
|
||||
RecentCMD string
|
||||
RecentCMDParams []string
|
||||
}
|
||||
|
||||
func NewFakeDB() *FakeDB {
|
||||
return &FakeDB{LastKeyboardMID: -1}
|
||||
}
|
||||
|
||||
func (db *FakeDB) LastKeyboardMsgID() (int, bool) {
|
||||
if db.LastKeyboardMID == -1 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return db.LastKeyboardMID, true
|
||||
}
|
||||
|
||||
func (db *FakeDB) SetLastKeyboardMsgID(msgID int) {
|
||||
db.LastKeyboardMID = msgID
|
||||
}
|
||||
|
||||
func (db *FakeDB) DelLastKeyboardMsgID() {
|
||||
db.LastKeyboardMID = -1
|
||||
}
|
||||
|
||||
func (db *FakeDB) InputExpectation() *tg.Cmd {
|
||||
return db.InputExpectationCMD
|
||||
}
|
||||
|
||||
func (db *FakeDB) SetInputExpectation(cmd tg.Cmd) {
|
||||
db.InputExpectationCMD = &cmd
|
||||
}
|
||||
|
||||
func (db *FakeDB) DelInputExpectation() {
|
||||
db.InputExpectationCMD = nil
|
||||
}
|
||||
|
||||
func (db *FakeDB) HashOrPathByMsgID(msgID int) (string, bool) {
|
||||
return db.HashOrPathByMID, db.HashOrPathByMID != ""
|
||||
}
|
||||
|
||||
func (db *FakeDB) SetHashOrPathByMsgID(msgID int, value string) {
|
||||
db.HashOrPathByMID = value
|
||||
}
|
||||
|
||||
func (db *FakeDB) RecentCommand() (string, bool) {
|
||||
return db.RecentCMD, db.RecentCMD != ""
|
||||
}
|
||||
|
||||
func (db *FakeDB) SetRecentCommand(cmd string) {
|
||||
db.RecentCMD = cmd
|
||||
}
|
||||
|
||||
func (db *FakeDB) RecentCommandParams() ([]string, bool) {
|
||||
return db.RecentCMDParams, len(db.RecentCMDParams) > 0
|
||||
}
|
||||
|
||||
func (db *FakeDB) SetRecentCommandParams(params []string) {
|
||||
db.RecentCMDParams = params
|
||||
}
|
||||
|
||||
func (db *FakeDB) AddImgMsgID(msgID int) {
|
||||
}
|
||||
|
||||
func (db *FakeDB) ImgMsgID() ([]int, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (db *FakeDB) DelImgMsgID() {
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package db
|
||||
+655
@@ -0,0 +1,655 @@
|
||||
// Package fs provides a simple interface for files manipulations.
|
||||
// Bot users should have all their artefacts saved in cross-platform
|
||||
// plain text files, that's why we use good old-fashioned filesystem.
|
||||
// Each user should have its own isolated root folder.
|
||||
// TODO maybe make ... access for all methods? So we can use both paths and segments
|
||||
// Why not BasePathFS?
|
||||
package fs
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/pkg/txt"
|
||||
)
|
||||
|
||||
var (
|
||||
NewUserFS = newUserFS
|
||||
|
||||
LogRename = func(time int64, oldPath, newPath string) {} // callback to track renames
|
||||
LogDelete = func(time int64, path string) {} // callback to track deletes
|
||||
|
||||
ErrQuotaExceeded = errors.New("storage quota exceeded")
|
||||
ErrUnsafePath = errors.New("unsafe path, possible security issue")
|
||||
ErrCannotUnhash = errors.New("cannot unhash, maybe the file is missing")
|
||||
)
|
||||
|
||||
const (
|
||||
DirUserRoot = "/"
|
||||
DirArchive = "archive"
|
||||
DirMedia = "media"
|
||||
DirJournal = "journal"
|
||||
DirHabits = "habits"
|
||||
DirInsights = "insights"
|
||||
|
||||
ChatFilename = "Chat.md"
|
||||
LaterFilename = "Later.md"
|
||||
DoneFilename = "Done.md"
|
||||
|
||||
ShopFilename = "Shop.md"
|
||||
WatchFilename = "Watch.md"
|
||||
ReadFilename = "Read.md"
|
||||
|
||||
PomodoroTask = "Finished a break"
|
||||
|
||||
MDExt = ".md"
|
||||
|
||||
minSearchSimilarity = 70
|
||||
)
|
||||
|
||||
// FS allows us to manipulate user files. We can use different
|
||||
// backends, like an in-memory backend, which we use for testing.
|
||||
// Check out types implementing afero.Fs for all available backends.
|
||||
type FS struct {
|
||||
rootPath string
|
||||
backend afero.Fs
|
||||
quotaKB int64
|
||||
}
|
||||
|
||||
// File represents a file or directory
|
||||
type File struct {
|
||||
Name string // Filename with extension
|
||||
Hash string
|
||||
DisplayName string
|
||||
Ctime int64
|
||||
IsMultiline bool
|
||||
IsDir bool
|
||||
ParentDir string
|
||||
}
|
||||
|
||||
// newUserFS creates a new FS for a specific user with os.FS backend.
|
||||
func newUserFS(userID int64) (*FS, error) {
|
||||
userAbsPath := path.Join(config.ServerCfg.StorageDir, txt.I64(userID))
|
||||
backend := afero.NewOsFs()
|
||||
|
||||
quotaKB := config.ServerCfg.StorageQuotaKB
|
||||
if isUnlimitedQuota(userID, config.ServerCfg.UnlimitedQuotaIDs) {
|
||||
quotaKB = 0
|
||||
}
|
||||
|
||||
return NewFS(userAbsPath, backend, quotaKB)
|
||||
}
|
||||
|
||||
func NewFS(absRootPath string, backend afero.Fs, quotaKB ...int64) (*FS, error) {
|
||||
exists, err := afero.Exists(backend, absRootPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new fs: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
err = backend.Mkdir(absRootPath, 0o755)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new fs: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var q int64
|
||||
if len(quotaKB) > 0 {
|
||||
q = quotaKB[0]
|
||||
}
|
||||
|
||||
return &FS{absRootPath, backend, q}, nil
|
||||
}
|
||||
|
||||
func NewFile(name, hash, displayName string, ctime int64, isMultiline, isDir bool, parentDir string) File {
|
||||
return File{
|
||||
Name: name,
|
||||
Hash: hash,
|
||||
DisplayName: displayName,
|
||||
Ctime: ctime,
|
||||
IsMultiline: isMultiline,
|
||||
IsDir: isDir,
|
||||
ParentDir: parentDir,
|
||||
}
|
||||
}
|
||||
|
||||
// CreateDirsIfNotExist creates specified directories for a user if they do not exist.
|
||||
// If dirs are not specified, it creates default directories.
|
||||
func (fs FS) CreateDirsIfNotExist(dirs ...string) error {
|
||||
for _, dir := range dirs {
|
||||
if dir == DirUserRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
userPath := path.Join(fs.rootPath, dir)
|
||||
exists, err := afero.Exists(fs.backend, userPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create default dirs: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
err = fs.backend.Mkdir(userPath, 0o755)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create default dirs: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs FS) CreateSystemDirs() error {
|
||||
return fs.CreateDirsIfNotExist(DirArchive, DirMedia, DirJournal)
|
||||
}
|
||||
|
||||
func (fs FS) Exists(dir, filename string) (bool, error) {
|
||||
filePath, err := fs.SafePath(dir, filename)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("exists: unsafe path '%s': %w", filepath.Join(dir, filename), ErrUnsafePath)
|
||||
}
|
||||
|
||||
exists, err := afero.Exists(fs.backend, filePath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("exists: can't check whether the file '%s'/'%s' exists: %w", dir, filename, err)
|
||||
}
|
||||
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (fs FS) Read(dir, filename string) (string, error) {
|
||||
filePath, err := fs.SafePath(dir, filename)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fs read: unsafe filePath dir: '%s', filename: '%s': %w", dir, filename, ErrUnsafePath)
|
||||
}
|
||||
|
||||
content, err := afero.ReadFile(fs.backend, filePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fs read: can't read file '%s': %w", filePath, err)
|
||||
}
|
||||
|
||||
return string(content), nil
|
||||
}
|
||||
|
||||
func (fs FS) Write(dir, filename, content string) error {
|
||||
filePath, err := fs.SafePath(dir, filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fs write: unsafe filePath '%s': %w", filepath.Join(dir, filename), ErrUnsafePath)
|
||||
}
|
||||
|
||||
dirs := strings.Split(filePath, "/")
|
||||
dirs = dirs[:len(dirs)-1]
|
||||
pathToDir := strings.Join(dirs, "/")
|
||||
if err := fs.backend.MkdirAll(pathToDir, 0o755); err != nil {
|
||||
return fmt.Errorf("fs write: can't create dirs '%s': %w", pathToDir, err)
|
||||
}
|
||||
|
||||
// Track old size for quota accounting.
|
||||
var oldSize int64
|
||||
if info, err := fs.backend.Stat(filePath); err == nil {
|
||||
oldSize = info.Size()
|
||||
}
|
||||
|
||||
newSize := int64(len(content))
|
||||
if err := checkQuota(fs.rootPath, fs.backend, fs.quotaKB, newSize-oldSize); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Append mode for forwards?
|
||||
if err := afero.WriteFile(fs.backend, filePath, []byte(content), 0o644); err != nil {
|
||||
return fmt.Errorf("fs write to '%s/%s': %w", dir, filename, err)
|
||||
}
|
||||
|
||||
recordQuotaUsage(fs.rootPath, newSize-oldSize)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs FS) MakeDir(dir string) error {
|
||||
filePath, err := fs.SafePath(dir, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("fs make dir: unsafe filePath '%s': %w", filePath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
err = fs.backend.Mkdir(filePath, 0o755)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fs can't make dir: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs FS) Del(dir, filename string) error {
|
||||
filePath, err := fs.SafePath(dir, filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fs del file: unsafe filePath '%s': %w", filePath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
var fileSize int64
|
||||
if info, err := fs.backend.Stat(filePath); err == nil {
|
||||
fileSize = info.Size()
|
||||
}
|
||||
|
||||
err = fs.backend.Remove(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fs file: can't remove '%s': %w", filePath, err)
|
||||
}
|
||||
|
||||
recordQuotaUsage(fs.rootPath, -fileSize)
|
||||
|
||||
// filePath is already absolute (SafePath joins rootPath in).
|
||||
LogDelete(time.Now().Unix(), filePath)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs FS) Rename(oldDir, oldFilename, newDir, newFilename string) error {
|
||||
oldPath, err := fs.SafePath(oldDir, oldFilename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fs can't rename from '%s': %w", oldPath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
newPath, err := fs.SafePath(newDir, newFilename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fs can't rename to '%s': %w", newPath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
err = fs.CreateDirsIfNotExist(oldDir, newDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fs can't rename: %w", err)
|
||||
}
|
||||
|
||||
err = fs.backend.Rename(oldPath, newPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't rename from '%s' to '%s': %w", oldPath, newPath, err)
|
||||
}
|
||||
|
||||
// Log renaming.
|
||||
ctime, err := fs.Ctime(newDir, newFilename)
|
||||
// Nothing terrible will happen if we don't log a rename. The client would just have duplicate files.
|
||||
if err == nil {
|
||||
absOldPath := path.Join(fs.rootPath, oldDir, oldFilename)
|
||||
absNewPath := path.Join(fs.rootPath, newDir, newFilename)
|
||||
LogRename(ctime, absOldPath, absNewPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs FS) Unhash(dir, filenameHash string) (string, error) {
|
||||
if dir == DirUserRoot && filenameHash == DirUserRoot {
|
||||
return DirUserRoot, nil
|
||||
}
|
||||
|
||||
filenames, err := fs.FilesAndDirs(dir)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("can't unhash: %w", err)
|
||||
}
|
||||
for _, file := range filenames {
|
||||
if strings.HasPrefix(fs.md5(file.Name), filenameHash) {
|
||||
return file.Name, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback, treat hash as filename
|
||||
for _, file := range filenames {
|
||||
if strings.HasPrefix(file.Name, filenameHash) {
|
||||
return file.Name, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("can't unhash '%s' in '%s': %w", filenameHash, dir, ErrCannotUnhash)
|
||||
}
|
||||
|
||||
func (fs FS) FilesAndDirs(dir string) ([]File, error) {
|
||||
userPath, err := fs.SafePath(dir, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't get files for '%s': %w", path.Join(fs.rootPath, dir), ErrUnsafePath)
|
||||
}
|
||||
|
||||
entries, err := afero.ReadDir(fs.backend, userPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
// Create folders only on write-actions
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("can't get files for '%s': %w", path.Join(fs.rootPath, dir), err)
|
||||
}
|
||||
|
||||
var files []File
|
||||
ignoredFiles := []string{".", "..", ".obsidian", ".gitignore", ".DS_Store", ".git"}
|
||||
for _, entry := range entries {
|
||||
if slices.Contains(ignoredFiles, entry.Name()) {
|
||||
continue
|
||||
}
|
||||
|
||||
file := NewFile(
|
||||
entry.Name(),
|
||||
Hash(entry.Name()),
|
||||
DisplayName(entry.Name()),
|
||||
Ctime(entry),
|
||||
entry.Size() > 0,
|
||||
entry.IsDir(),
|
||||
dir,
|
||||
)
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (fs FS) Dirs() ([]File, error) {
|
||||
files, err := fs.FilesAndDirs(DirUserRoot)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't get dirs: %w", err)
|
||||
}
|
||||
|
||||
var dirs []File
|
||||
for _, file := range files {
|
||||
filePath, err := fs.SafePath(DirUserRoot, file.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't get dirs: unsafe path '%s': %w", filePath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
isDir, err := afero.IsDir(fs.backend, filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't get dirs: %w", err)
|
||||
}
|
||||
if !isDir {
|
||||
continue
|
||||
}
|
||||
|
||||
dirs = append(dirs, file)
|
||||
}
|
||||
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
func (fs FS) IsMultiline(dir, filename string) (bool, error) {
|
||||
content, err := fs.Read(dir, filename)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("can't check for multiline: %w", err)
|
||||
}
|
||||
content = strings.TrimSpace(content)
|
||||
|
||||
return len(content) > 0, nil
|
||||
}
|
||||
|
||||
func (fs FS) md5(filename string) string {
|
||||
hash := md5.Sum([]byte(filename))
|
||||
return hex.EncodeToString(hash[:])[:11]
|
||||
}
|
||||
|
||||
func Filename(header string) string {
|
||||
return txt.Ucfirst(header) + MDExt
|
||||
}
|
||||
|
||||
func IsChecklistItem(filename string) bool {
|
||||
validChecklistItem := regexp.MustCompile(`^-.*?-(.+)`)
|
||||
|
||||
return validChecklistItem.MatchString(filename)
|
||||
}
|
||||
|
||||
// SearchFilesByName performs search among all user .md files
|
||||
// Allowed query formats:
|
||||
// "directory" - return all notes from directories prefixed by this directory
|
||||
// "directory note_name" - search for this note_name in all matching directories
|
||||
// "note_name" - search for this note_name across all directories
|
||||
// "" - return all the notes
|
||||
func (fs FS) SearchFilesByName(query string) ([]File, error) {
|
||||
query = strings.ToLower(strings.TrimSpace(query))
|
||||
// Check for directory traversal attack
|
||||
if strings.Contains(query, "/") {
|
||||
return nil, fmt.Errorf("search notes: unsafe query '%s': %w", query, ErrUnsafePath)
|
||||
}
|
||||
|
||||
var supposedDir, search string
|
||||
dirExists, err := fs.Exists(DirUserRoot, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search notes: %w", err)
|
||||
}
|
||||
if dirExists {
|
||||
supposedDir = query
|
||||
} else {
|
||||
parts := strings.SplitN(query, " ", 2)
|
||||
supposedDir = parts[0]
|
||||
if len(parts) > 1 {
|
||||
search = strings.TrimSpace(parts[1])
|
||||
}
|
||||
}
|
||||
|
||||
rootPath, err := fs.SafePath(DirUserRoot, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search notes: %w", err)
|
||||
}
|
||||
|
||||
// Walk the whole tree once - up to 10 levels deep - and collect .md
|
||||
// files. Filtering by dir prefix happens after the walk.
|
||||
var notes []File
|
||||
_ = afero.Walk(fs.backend, rootPath, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil || info == nil {
|
||||
return nil
|
||||
}
|
||||
base := filepath.Base(p)
|
||||
if strings.HasPrefix(base, ".") {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if info.IsDir() {
|
||||
rel, _ := filepath.Rel(rootPath, p)
|
||||
if rel != "." && strings.Count(rel, string(filepath.Separator)) >= 10 {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if filepath.Ext(base) != MDExt {
|
||||
return nil
|
||||
}
|
||||
relativeToUserRootPath, _ := filepath.Rel(rootPath, filepath.Dir(p))
|
||||
if relativeToUserRootPath == "." || relativeToUserRootPath == "" {
|
||||
relativeToUserRootPath = DirUserRoot
|
||||
}
|
||||
notes = append(notes, NewFile(
|
||||
base, Hash(base), DisplayName(base),
|
||||
Ctime(info), info.Size() > 0, false, relativeToUserRootPath,
|
||||
))
|
||||
return nil
|
||||
})
|
||||
notes = OnlyUserMDFiles(notes)
|
||||
|
||||
// Restrict to the matching top-level dir if the user typed one.
|
||||
if supposedDir != "" {
|
||||
var pruned []File
|
||||
for _, n := range notes {
|
||||
top := strings.SplitN(n.ParentDir, "/", 2)[0]
|
||||
if top == DirUserRoot {
|
||||
top = ""
|
||||
}
|
||||
if strings.HasPrefix(top, supposedDir) {
|
||||
pruned = append(pruned, n)
|
||||
}
|
||||
}
|
||||
if len(pruned) > 0 {
|
||||
notes = pruned
|
||||
} else {
|
||||
// No dir matched - fall back to a name search across all notes.
|
||||
search = query
|
||||
}
|
||||
}
|
||||
notes = SortByCtimeDesc(notes)
|
||||
|
||||
var matchedNotes []File
|
||||
for _, note := range notes {
|
||||
isWildcard := len(search) == 0
|
||||
isSubstring := strings.Contains(strings.ToLower(note.DisplayName), search)
|
||||
isSimilar := txt.Similar(strings.ToLower(note.DisplayName), search) > minSearchSimilarity
|
||||
if isWildcard || isSubstring || isSimilar {
|
||||
matchedNotes = append(matchedNotes, note)
|
||||
}
|
||||
}
|
||||
|
||||
return matchedNotes, nil
|
||||
}
|
||||
|
||||
// TODO check if safe
|
||||
// Touch updates an existing file's access and modification times.
|
||||
// If there's no such file it creates an empty file.
|
||||
func (fs FS) Touch(dir, filename string) error {
|
||||
filePath, err := fs.SafePath(dir, filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("touch: unsafe path '%s': %w", filePath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
exists, err := fs.Exists(dir, filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("touch: %w", err)
|
||||
}
|
||||
|
||||
if exists {
|
||||
err = fs.backend.Chtimes(filePath, time.Now(), time.Now())
|
||||
if err != nil {
|
||||
return fmt.Errorf("touch: can't update file's ctime: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
err = fs.Write(dir, filename, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("touch: can't create empty file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ctime returns the change time of a file as Unix timestamp in milliseconds.
|
||||
// It updates on file creation, modification, metadata changes, moving to a different dir, renames.
|
||||
func (fs FS) Ctime(dir, filename string) (int64, error) {
|
||||
filePath, err := fs.SafePath(dir, filename)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("fs file: unsafe filePath '%s': %w", filePath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
info, err := fs.backend.Stat(filePath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("fs file: can't stat file '%s': %w", filePath, err)
|
||||
}
|
||||
|
||||
return Ctime(info), nil
|
||||
}
|
||||
|
||||
// Mtime returns the modification time of a file as Unix timestamp in milliseconds.
|
||||
// It only updates on file modification.
|
||||
func (fs FS) Mtime(dir, filename string) (int64, error) {
|
||||
filePath, err := fs.SafePath(dir, filename)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("fs mtime: unsafe filePath '%s': %w", filePath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
info, err := fs.backend.Stat(filePath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("fs mtime: can't stat file '%s': %w", filePath, err)
|
||||
}
|
||||
|
||||
return Mtime(info), nil
|
||||
}
|
||||
|
||||
// Mtimes recursively scans a directory and returns the mtime
|
||||
// for all files with specified extension as Unix timestamps.
|
||||
// Returns [relPath] => ctime
|
||||
// TODO add tests
|
||||
func (fs FS) Mtimes(root string, extensions ...string) (map[string]int64, error) {
|
||||
rootPath, err := fs.SafePath(root, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fs mtimes: unsafe rootPath '%s': %w", rootPath, ErrUnsafePath)
|
||||
}
|
||||
|
||||
mtimes := make(map[string]int64)
|
||||
err = afero.Walk(fs.backend, rootPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
base := filepath.Base(path)
|
||||
// Skip hidden files.
|
||||
if strings.HasPrefix(base, ".") && path != rootPath {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only process specified file extension.
|
||||
if len(extensions) > 0 {
|
||||
ext := filepath.Ext(path)
|
||||
if !slices.Contains(extensions, ext) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// TODO what if a file inside folder?
|
||||
relPath, err := filepath.Rel(rootPath, path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if relPath == "" {
|
||||
relPath = "."
|
||||
}
|
||||
|
||||
mtimes[relPath] = Mtime(info)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return mtimes, nil
|
||||
}
|
||||
|
||||
// SafePath returns safe path to a file or directory, error if the path is unsafe.
|
||||
// Sanitize Early, call SanitizeFilename
|
||||
// as soon as you get on dir and filename from user input
|
||||
// TODO test all FS' public the methods for unsafePath traversal
|
||||
// TODO after you cover everything with the tests, we may remove this method
|
||||
// because we build our own paths (???)
|
||||
// TODO release remove error?
|
||||
// isSafe doesn't eval symlinks, so an attacker can create a symlink to a file
|
||||
// outside the rootPath. If we use filepath.EvalSymlinks to expand symlinks and
|
||||
// check the real path for safety - we are still prone to TOCTOU (time-of-check to time-of-use)
|
||||
// attacks due to the race condition. The only real way to prevent this is to disallow symlinks
|
||||
// at the OS level. We can do this by mounting a folder with nosymfollow flag, see README.md.
|
||||
func (fs FS) SafePath(dir, filename string) (string, error) {
|
||||
var relativePath string
|
||||
if dir == "/" {
|
||||
if filename == "" {
|
||||
// Just the root directory
|
||||
return fs.rootPath, nil
|
||||
}
|
||||
relativePath = filename
|
||||
} else {
|
||||
relativePath = filepath.Join(dir, filename)
|
||||
}
|
||||
|
||||
if !filepath.IsLocal(relativePath) {
|
||||
return "", ErrUnsafePath
|
||||
}
|
||||
|
||||
return filepath.Join(fs.rootPath, relativePath), nil
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
//go:build darwin
|
||||
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var Ctime = func(fi os.FileInfo) int64 {
|
||||
stat := fi.Sys().(*syscall.Stat_t)
|
||||
|
||||
return (stat.Ctimespec.Sec*1_000_000_000 + stat.Ctimespec.Nsec) / 1000 // Look for CONFIG_HZ in README.md
|
||||
}
|
||||
|
||||
var Mtime = func(fi os.FileInfo) int64 {
|
||||
stat := fi.Sys().(*syscall.Stat_t)
|
||||
|
||||
return (stat.Mtimespec.Sec*1_000_000_000 + stat.Mtimespec.Nsec) / 1000 // Look for CONFIG_HZ in README.md
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/pkg/txt"
|
||||
)
|
||||
|
||||
// ForbiddenChars hold replacements for characters
|
||||
// not allowed in some envs like Windows, PWA apps.
|
||||
// Under Linux and other Unix-related systems,
|
||||
// there are only two characters that cannot
|
||||
// appear in the name of a file or directory,
|
||||
// and those are NUL '\0' and slash '/'.
|
||||
var ForbiddenChars = map[string]string{
|
||||
"<": "<",
|
||||
">": ">",
|
||||
":": "꞉",
|
||||
"\"": "″",
|
||||
"|": "⼁",
|
||||
"\\": "\",
|
||||
"?": "?",
|
||||
"*": "﹡",
|
||||
// Forbidden on Unix.
|
||||
"\x00": "",
|
||||
"/": "/",
|
||||
}
|
||||
|
||||
func SanitizeFilename(filename string) string {
|
||||
for forbidden, safe := range ForbiddenChars {
|
||||
filename = strings.ReplaceAll(filename, forbidden, safe)
|
||||
}
|
||||
|
||||
return filename
|
||||
}
|
||||
|
||||
func UnsanitizeFilename(filename string) string {
|
||||
for forbidden, safe := range ForbiddenChars {
|
||||
if safe == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
filename = strings.ReplaceAll(filename, safe, forbidden)
|
||||
}
|
||||
|
||||
return filename
|
||||
}
|
||||
|
||||
func DisplayName(filename string) string {
|
||||
return txt.Ucfirst(strings.TrimSuffix(strings.TrimSpace(filename), MDExt))
|
||||
}
|
||||
|
||||
func Hash(filename string) string {
|
||||
hash := md5.Sum([]byte(filename))
|
||||
return hex.EncodeToString(hash[:])[:11]
|
||||
}
|
||||
|
||||
// ShortHash returns a short hash of the filename
|
||||
// Telegram only allows 64 bytes in callback_data,
|
||||
// so if we have 3 params we should use shortHash.
|
||||
// More collisions are possible, but it's a trade-off.
|
||||
func ShortHash(filename string) string {
|
||||
hash := md5.Sum([]byte(filename))
|
||||
return hex.EncodeToString(hash[:])[:5]
|
||||
}
|
||||
|
||||
func ExcludeChecklists(dirs []File) []File {
|
||||
var newDirs []File
|
||||
for _, dir := range dirs {
|
||||
isChecklist := strings.HasPrefix(dir.Name, "_") && strings.HasSuffix(dir.Name, "_")
|
||||
if isChecklist {
|
||||
continue
|
||||
}
|
||||
|
||||
newDirs = append(newDirs, dir)
|
||||
}
|
||||
|
||||
return newDirs
|
||||
}
|
||||
|
||||
func ExcludeSystemDirs(dirs []File) []File {
|
||||
var newDirs []File
|
||||
for _, dir := range dirs {
|
||||
if slices.Contains([]string{DirMedia, DirArchive, DirJournal, DirInsights, "img"}, dir.Name) {
|
||||
continue
|
||||
}
|
||||
|
||||
newDirs = append(newDirs, dir)
|
||||
}
|
||||
|
||||
return newDirs
|
||||
}
|
||||
|
||||
func ExcludeConfig(files []File) []File {
|
||||
var newFiles []File
|
||||
for _, file := range files {
|
||||
if file.Name == config.ServerCfg.ConfigFilename && file.ParentDir == DirUserRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
newFiles = append(newFiles, file)
|
||||
}
|
||||
|
||||
return newFiles
|
||||
}
|
||||
|
||||
func OnlyNoteDirs(dirs []File) []File {
|
||||
return ExcludeSystemDirs(ExcludeChecklists(dirs))
|
||||
}
|
||||
|
||||
func OnlyChecklists(dirs []File) []File {
|
||||
entries := OnlyFiles(dirs)
|
||||
|
||||
var checklists []File
|
||||
for _, entry := range entries {
|
||||
if filepath.Ext(entry.Name) != MDExt {
|
||||
continue
|
||||
}
|
||||
filename := strings.TrimSuffix(entry.Name, MDExt)
|
||||
hasChecklistPostfix := strings.HasSuffix(filename, "_")
|
||||
if hasChecklistPostfix || slices.Contains([]string{
|
||||
WatchFilename,
|
||||
ReadFilename,
|
||||
ShopFilename,
|
||||
}, entry.Name) {
|
||||
checklists = append(checklists, entry)
|
||||
}
|
||||
}
|
||||
|
||||
return checklists
|
||||
}
|
||||
|
||||
func OnlyUserMDFiles(entries []File) []File {
|
||||
systemFiles := []string{
|
||||
ChatFilename,
|
||||
LaterFilename,
|
||||
DoneFilename,
|
||||
ChatFilename,
|
||||
ShopFilename,
|
||||
WatchFilename,
|
||||
ReadFilename,
|
||||
}
|
||||
|
||||
var files []File
|
||||
for _, file := range entries {
|
||||
if file.IsDir {
|
||||
continue
|
||||
}
|
||||
if filepath.Ext(file.Name) != MDExt {
|
||||
continue
|
||||
}
|
||||
if slices.Contains(systemFiles, file.Name) {
|
||||
continue
|
||||
}
|
||||
files = append(files, file)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func OnlyFiles(entries []File) []File {
|
||||
var files []File
|
||||
for _, file := range entries {
|
||||
if file.IsDir {
|
||||
continue
|
||||
}
|
||||
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
func OnlyDirs(entries []File) []File {
|
||||
var dirs []File
|
||||
for _, file := range entries {
|
||||
if !file.IsDir {
|
||||
continue
|
||||
}
|
||||
|
||||
dirs = append(dirs, file)
|
||||
}
|
||||
|
||||
return dirs
|
||||
}
|
||||
|
||||
// OnlyUserDirs returns only directories that look like user IDs
|
||||
func OnlyUserDirs(entries []File) []File {
|
||||
var dirs []File
|
||||
for _, file := range entries {
|
||||
if !file.IsDir {
|
||||
continue
|
||||
}
|
||||
if _, err := strconv.Atoi(file.Name); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
dirs = append(dirs, file)
|
||||
}
|
||||
|
||||
return dirs
|
||||
}
|
||||
|
||||
func OnlyFilenames(entries []File) []string {
|
||||
var filenames []string
|
||||
for _, entry := range entries {
|
||||
filenames = append(filenames, entry.Name)
|
||||
}
|
||||
|
||||
return filenames
|
||||
}
|
||||
|
||||
func SortByCtimeDesc(entries []File) []File {
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return entries[i].Ctime > entries[j].Ctime
|
||||
})
|
||||
|
||||
return entries
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
//go:build linux
|
||||
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// Ctime returns the creation time of a file in microseconds since epoch.
|
||||
var Ctime = func(fi os.FileInfo) int64 {
|
||||
stat := fi.Sys().(*syscall.Stat_t)
|
||||
|
||||
return (stat.Ctim.Sec*1_000_000_000 + stat.Ctim.Nsec) / 1000 // Look for CONFIG_HZ in README.md
|
||||
}
|
||||
|
||||
var Mtime = func(fi os.FileInfo) int64 {
|
||||
stat := fi.Sys().(*syscall.Stat_t)
|
||||
|
||||
return (stat.Mtim.Sec*1_000_000_000 + stat.Mtim.Nsec) / 1000 // Look for CONFIG_HZ in README.md
|
||||
}
|
||||
@@ -0,0 +1,932 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Ctime = func(fi os.FileInfo) int64 {
|
||||
return 0
|
||||
}
|
||||
Mtime = func(fi os.FileInfo) int64 { return 0 }
|
||||
}
|
||||
|
||||
func TestIsChecklistItem(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
r.False(IsChecklistItem("-checklist-"))
|
||||
r.True(IsChecklistItem("-checklist-item"))
|
||||
}
|
||||
|
||||
func TestDisplayName(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
title := DisplayName("filename")
|
||||
r.Equal("Filename", title)
|
||||
}
|
||||
|
||||
func TestDisplayNameWithSpace(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
title := DisplayName(" filename ")
|
||||
r.Equal("Filename", title)
|
||||
}
|
||||
|
||||
func TestMD5(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
res := fs.md5("First task.md")
|
||||
|
||||
r.Equal("0824149b387", res)
|
||||
}
|
||||
|
||||
func TestExcludeChecklists(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
noChecklists := ExcludeChecklists([]File{{Name: "not-a-checklist"}, {Name: "_checklist_"}})
|
||||
|
||||
r.Equal([]File{{Name: "not-a-checklist"}}, noChecklists)
|
||||
}
|
||||
|
||||
func TestExcludeSystemDirs(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
noChecklists := ExcludeSystemDirs([]File{{Name: "not-a-system-dir"}, {Name: "media"}, {Name: "archive"}, {Name: "journal"}})
|
||||
|
||||
r.Equal([]File{{Name: "not-a-system-dir"}}, noChecklists)
|
||||
}
|
||||
|
||||
func TestIsMultiline(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "First task.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
isMultiline, err := fs.IsMultiline("today", "First task.md")
|
||||
r.NoError(err)
|
||||
r.False(isMultiline)
|
||||
|
||||
err = fs.Write("today", "Second task.md", "c")
|
||||
r.NoError(err)
|
||||
|
||||
isMultiline, err = fs.IsMultiline("today", "Second task.md")
|
||||
r.NoError(err)
|
||||
r.True(isMultiline)
|
||||
|
||||
err = fs.Write("today", "Third task.md", " \n ")
|
||||
r.NoError(err)
|
||||
|
||||
isMultiline, err = fs.IsMultiline("today", "Third task.md")
|
||||
r.NoError(err)
|
||||
r.False(isMultiline)
|
||||
}
|
||||
|
||||
func TestGetFilesInDir(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "First task.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
files, err := fs.FilesAndDirs("today")
|
||||
r.NoError(err)
|
||||
r.Len(files, 1)
|
||||
r.Equal("First task.md", files[0].Name)
|
||||
}
|
||||
|
||||
func TestCreateBaseDirs(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, err := NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
r.NoError(fs.CreateSystemDirs())
|
||||
|
||||
err = fs.CreateSystemDirs()
|
||||
r.NoError(err)
|
||||
|
||||
dirs, err := fs.FilesAndDirs("/")
|
||||
r.NoError(err)
|
||||
dirs = OnlyDirs(dirs)
|
||||
dirNames := OnlyFilenames(dirs)
|
||||
|
||||
r.ElementsMatch([]string{"archive", "media", "journal"}, dirNames)
|
||||
}
|
||||
|
||||
func TestSortByCtimeDesc(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
saved := Ctime
|
||||
defer func() {
|
||||
Ctime = saved
|
||||
}()
|
||||
Ctime = func(fi os.FileInfo) int64 {
|
||||
if fi.Name() == "b.md" {
|
||||
return 1
|
||||
}
|
||||
|
||||
return 2
|
||||
}
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "b.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Write("today", "a.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
entries, err := fs.FilesAndDirs("today")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal([]string{"a.md", "b.md"}, OnlyFilenames(SortByCtimeDesc(entries)))
|
||||
}
|
||||
|
||||
func TestExcludeEverythingButUserDirs(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("", "a.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.MakeDir("dir")
|
||||
r.NoError(err)
|
||||
|
||||
entries, err := fs.FilesAndDirs("/")
|
||||
r.NoError(err)
|
||||
|
||||
dirs := OnlyDirs(ExcludeSystemDirs(entries))
|
||||
r.Len(dirs, 1)
|
||||
r.Equal("dir", dirs[0].Name)
|
||||
}
|
||||
|
||||
func TestOnlyFiles(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("", "a.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.MakeDir("dir")
|
||||
r.NoError(err)
|
||||
|
||||
entries, err := fs.FilesAndDirs("/")
|
||||
r.NoError(err)
|
||||
|
||||
dirs := OnlyUserMDFiles(entries)
|
||||
r.Len(dirs, 1)
|
||||
r.Equal("a.md", dirs[0].Name)
|
||||
}
|
||||
|
||||
func TestOnlyChecklists(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(fs.Write("today", "a.md", ""))
|
||||
r.NoError(fs.Write("/", "list_.md", ""))
|
||||
// A non-markdown file whose name happens to end in "_" must NOT be
|
||||
// treated as a checklist.
|
||||
r.NoError(fs.Write("/", "Molchanov_.mobi", ""))
|
||||
|
||||
entries, err := fs.FilesAndDirs("/")
|
||||
r.NoError(err)
|
||||
|
||||
dirs := OnlyChecklists(entries)
|
||||
r.Len(dirs, 1)
|
||||
r.Equal("list_.md", dirs[0].Name)
|
||||
}
|
||||
|
||||
func TestFSTouchNew(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
exists, err := fs.Exists("today", "a.md")
|
||||
r.NoError(err)
|
||||
r.False(exists)
|
||||
|
||||
err = fs.Touch("today", "a.md")
|
||||
r.NoError(err)
|
||||
|
||||
exists, err = fs.Exists("today", "a.md")
|
||||
r.NoError(err)
|
||||
r.True(exists)
|
||||
}
|
||||
|
||||
func TestFSTouchExisting(t *testing.T) {
|
||||
r := require.New(t)
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "a.md", "A")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Touch("today", "a.md")
|
||||
r.NoError(err)
|
||||
|
||||
content, err := fs.Read("today", "a.md")
|
||||
r.NoError(err)
|
||||
r.Equal("A", content)
|
||||
}
|
||||
|
||||
func TestFSGetAllNotesInMatchingDir(t *testing.T) {
|
||||
r := require.New(t)
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Touch("brain", "a.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("today", "b.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("non-matching-dir", "c.md")
|
||||
r.NoError(err)
|
||||
|
||||
notes, err := fs.SearchFilesByName("BRAIN")
|
||||
r.NoError(err)
|
||||
r.Len(notes, 1)
|
||||
r.Equal("a.md", notes[0].Name)
|
||||
}
|
||||
|
||||
func TestFSGetAllMatchingNotesInMatchingDir(t *testing.T) {
|
||||
r := require.New(t)
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Touch("brain", "a.md")
|
||||
|
||||
r.NoError(err)
|
||||
err = fs.Touch("brain", "b.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("today", "c.md")
|
||||
r.NoError(err)
|
||||
|
||||
notes, err := fs.SearchFilesByName("BRAIN A")
|
||||
r.NoError(err)
|
||||
r.Len(notes, 1)
|
||||
r.Equal("a.md", notes[0].Name)
|
||||
}
|
||||
|
||||
func TestFSGetAllNotesInAllMatchingDirs(t *testing.T) {
|
||||
r := require.New(t)
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Touch("brain", "a.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("brain", "b.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("today", "c.md")
|
||||
r.NoError(err)
|
||||
|
||||
notes, err := fs.SearchFilesByName("brain")
|
||||
r.NoError(err)
|
||||
r.Len(notes, 2)
|
||||
|
||||
var noteFilenames []string
|
||||
for _, note := range notes {
|
||||
noteFilenames = append(noteFilenames, note.Name)
|
||||
}
|
||||
|
||||
r.ElementsMatch([]string{"a.md", "b.md"}, noteFilenames)
|
||||
}
|
||||
|
||||
func TestFSGetAllMatchingNotesInAllMatchingDirs(t *testing.T) {
|
||||
r := require.New(t)
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Touch("brain", "a.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("brain", "ab.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("brain", "b.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("today", "c.md")
|
||||
r.NoError(err)
|
||||
|
||||
notes, err := fs.SearchFilesByName("brain a")
|
||||
r.NoError(err)
|
||||
r.Len(notes, 2)
|
||||
|
||||
var noteFilenames []string
|
||||
for _, note := range notes {
|
||||
noteFilenames = append(noteFilenames, note.Name)
|
||||
}
|
||||
|
||||
r.ElementsMatch([]string{"a.md", "ab.md"}, noteFilenames)
|
||||
}
|
||||
|
||||
func TestFSGetAllNotesInAllDirsForEmptyQuery(t *testing.T) {
|
||||
r := require.New(t)
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Touch("brain", "a.md")
|
||||
r.NoError(err)
|
||||
err = fs.Touch("b", "b.md")
|
||||
r.NoError(err)
|
||||
|
||||
notes, err := fs.SearchFilesByName("")
|
||||
r.NoError(err)
|
||||
r.Len(notes, 2)
|
||||
|
||||
var noteFilenames []string
|
||||
for _, note := range notes {
|
||||
noteFilenames = append(noteFilenames, note.Name)
|
||||
}
|
||||
|
||||
r.ElementsMatch([]string{"a.md", "b.md"}, noteFilenames)
|
||||
}
|
||||
|
||||
func TestFSPathTraversalAttack(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/abc", afero.NewMemMapFs())
|
||||
fs.rootPath = "/abc"
|
||||
|
||||
path, err := fs.SafePath("../root/.ssh/", "authorized_keys")
|
||||
r.Error(err)
|
||||
r.Equal("", path)
|
||||
|
||||
path, err = fs.SafePath("note", "../root/.ssh/authorized_keys")
|
||||
r.NoError(err)
|
||||
r.Equal("/abc/root/.ssh/authorized_keys", path)
|
||||
|
||||
path, err = fs.SafePath("note", "../../root/.ssh/authorized_keys")
|
||||
r.Error(err)
|
||||
r.Empty(path)
|
||||
}
|
||||
|
||||
func TestFSOnlyUserDirs(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, err := NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.MakeDir("str")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.MakeDir("123")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.MakeDir("123.56")
|
||||
r.NoError(err)
|
||||
|
||||
dirs, _ := fs.FilesAndDirs("/")
|
||||
userDirs := OnlyUserDirs(dirs)
|
||||
|
||||
r.Len(userDirs, 1)
|
||||
r.Equal("123", userDirs[0].Name)
|
||||
}
|
||||
|
||||
func TestIsSafeWrongRoot(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/a", afero.NewMemMapFs())
|
||||
p, err := fs.SafePath("b", "")
|
||||
r.NoError(err)
|
||||
r.Equal("/a/b", p)
|
||||
|
||||
}
|
||||
|
||||
func TestIsSafePathTraversalAttack(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/a", afero.NewMemMapFs())
|
||||
p, err := fs.SafePath("a/../b", "")
|
||||
r.NoError(err)
|
||||
r.Equal("/a/b", p)
|
||||
|
||||
p, err = fs.SafePath("/a/../../b", "")
|
||||
r.Error(err)
|
||||
r.Empty(p)
|
||||
|
||||
p, err = fs.SafePath("./a/../b", "")
|
||||
r.NoError(err)
|
||||
r.Equal("/a/b", p)
|
||||
|
||||
p, err = fs.SafePath("./a/../../b", "")
|
||||
r.Error(err)
|
||||
r.Empty(p)
|
||||
}
|
||||
|
||||
func TestSafePath(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
memFS := afero.NewMemMapFs()
|
||||
memFS.Mkdir("/app-secret", 0o755)
|
||||
|
||||
fs, _ := NewFS("/app", memFS)
|
||||
|
||||
p, err := fs.SafePath("subdir", "file")
|
||||
r.NoError(err)
|
||||
r.Equal("/app/subdir/file", p)
|
||||
|
||||
p, err = fs.SafePath("subdir", "../file")
|
||||
r.NoError(err)
|
||||
r.Equal("/app/file", p)
|
||||
|
||||
p, err = fs.SafePath("../app-secret", "file")
|
||||
r.Error(err)
|
||||
r.Equal("", p)
|
||||
|
||||
}
|
||||
|
||||
func TestIsSafePathTraversalAttackWithRelativePaths(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS(".", afero.NewMemMapFs())
|
||||
p, err := fs.SafePath("./a/../b", "")
|
||||
r.NoError(err)
|
||||
r.Equal("b", p)
|
||||
|
||||
p, err = fs.SafePath("./a/../../b", "")
|
||||
r.Error(err)
|
||||
r.Empty(p)
|
||||
}
|
||||
|
||||
func TestUnhashRootDirectory(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, err := NewFS(".", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
// TODO is it used at all? What a strange behaviour?
|
||||
unhashed, err := fs.Unhash("/", "/")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal("/", unhashed)
|
||||
}
|
||||
|
||||
func TestSanitizeFilename(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
r.Equal("ab", SanitizeFilename("a\x00b"))
|
||||
r.Equal("a/b", SanitizeFilename("a/b"))
|
||||
r.Equal("a\b", SanitizeFilename("a\\b"))
|
||||
r.Equal("a/b\", SanitizeFilename("\x00a\x00/b\\"))
|
||||
}
|
||||
|
||||
func TestUnsanitizeFilename(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
r.Equal("a/b", UnsanitizeFilename("a/b"))
|
||||
r.Equal("a\\b", UnsanitizeFilename("a\b"))
|
||||
r.Equal("a/b\\", UnsanitizeFilename("a/b\"))
|
||||
}
|
||||
|
||||
func TestExists(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "First task.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
exists, err := fs.Exists("today", "First task.md")
|
||||
r.NoError(err)
|
||||
r.True(exists)
|
||||
}
|
||||
|
||||
func TestExistsRoot(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "First task.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
exists, err := fs.Exists("/", "")
|
||||
r.NoError(err)
|
||||
r.True(exists)
|
||||
}
|
||||
|
||||
func TestWriteAndReadFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "test.md", "Test content")
|
||||
r.NoError(err)
|
||||
|
||||
content, err := fs.Read("today", "test.md")
|
||||
r.NoError(err)
|
||||
r.Equal("Test content", content)
|
||||
}
|
||||
|
||||
func TestWriteUnsafePath(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/1", afero.NewMemMapFs())
|
||||
err := fs.Write("../unsafe", "test.md", "Test content")
|
||||
r.Error(err)
|
||||
r.Contains(err.Error(), "unsafe filePath")
|
||||
}
|
||||
|
||||
func TestReadNonExistentFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
content, err := fs.Read("today", "nonexistent.md")
|
||||
r.Error(err)
|
||||
r.Equal("", content)
|
||||
}
|
||||
|
||||
func TestDelFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "delete.md", "To be deleted")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Del("today", "delete.md")
|
||||
r.NoError(err)
|
||||
|
||||
exists, err := fs.Exists("today", "delete.md")
|
||||
r.NoError(err)
|
||||
r.False(exists)
|
||||
}
|
||||
|
||||
func TestDelNonExistentFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Del("today", "nonexistent.md")
|
||||
r.Error(err)
|
||||
r.Contains(err.Error(), "can't remove")
|
||||
}
|
||||
|
||||
func TestRenameFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "oldname.md", "Old content")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Rename("today", "oldname.md", "today", "newname.md")
|
||||
r.NoError(err)
|
||||
|
||||
exists, err := fs.Exists("today", "newname.md")
|
||||
r.NoError(err)
|
||||
r.True(exists)
|
||||
|
||||
content, err := fs.Read("today", "newname.md")
|
||||
r.NoError(err)
|
||||
r.Equal("Old content", content)
|
||||
}
|
||||
|
||||
func TestRenameUnsafePath(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/1", afero.NewMemMapFs())
|
||||
err := fs.Write("today", "oldname.md", "Old content")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Rename("../unsafe", "oldname.md", "today", "newname.md")
|
||||
r.Error(err)
|
||||
r.Contains(err.Error(), "unsafe path")
|
||||
}
|
||||
|
||||
func TestMakeDir(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.MakeDir("newdir")
|
||||
r.NoError(err)
|
||||
|
||||
exists, err := fs.Exists("newdir", "")
|
||||
r.NoError(err)
|
||||
r.True(exists)
|
||||
}
|
||||
|
||||
func TestMakeDirUnsafePath(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/1", afero.NewMemMapFs())
|
||||
err := fs.MakeDir("../unsafe")
|
||||
r.Error(err)
|
||||
r.Contains(err.Error(), "unsafe path")
|
||||
}
|
||||
|
||||
func TestUnsafePathSanitization(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/1", afero.NewMemMapFs())
|
||||
|
||||
p, err := fs.SafePath("../", "test.md")
|
||||
r.Error(err)
|
||||
r.Empty(p)
|
||||
|
||||
p, err = fs.SafePath("safe", "../unsafe.md")
|
||||
r.NoError(err)
|
||||
r.Equal("/1/unsafe.md", p)
|
||||
}
|
||||
|
||||
func TestUnhash(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Touch("today", "hashedfile.md")
|
||||
r.NoError(err)
|
||||
|
||||
hash := fs.md5("hashedfile.md")
|
||||
unhashed, err := fs.Unhash("today", hash)
|
||||
r.NoError(err)
|
||||
r.Equal("hashedfile.md", unhashed)
|
||||
}
|
||||
|
||||
func TestUnhashNonExistentFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
_, err := fs.Unhash("today", "nonexistenthash")
|
||||
r.Error(err)
|
||||
r.Contains(err.Error(), "can't unhash")
|
||||
}
|
||||
|
||||
func TestSanitizeAndUnsanitizeFilename(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
sanitized := SanitizeFilename("test/file:name\\with/special\\chars")
|
||||
r.Equal("test/file꞉name\with/special\chars", sanitized)
|
||||
|
||||
unsanitized := UnsanitizeFilename(sanitized)
|
||||
r.Equal("test/file:name\\with/special\\chars", unsanitized)
|
||||
}
|
||||
|
||||
func TestFilesAndDirs(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.MakeDir("dir1")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Write("dir1", "file1.md", "File content")
|
||||
r.NoError(err)
|
||||
|
||||
files, err := fs.FilesAndDirs("dir1")
|
||||
r.NoError(err)
|
||||
r.Len(files, 1)
|
||||
r.Equal("file1.md", files[0].Name)
|
||||
}
|
||||
|
||||
func TestDirs(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.MakeDir("dir1")
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Write("dir1", "file1.md", "File content")
|
||||
r.NoError(err)
|
||||
|
||||
dirs, err := fs.Dirs()
|
||||
r.NoError(err)
|
||||
r.Len(dirs, 1)
|
||||
r.Equal("dir1", dirs[0].Name)
|
||||
}
|
||||
|
||||
func TestTouchFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, _ := NewFS("/", afero.NewMemMapFs())
|
||||
err := fs.Touch("today", "touchfile.md")
|
||||
r.NoError(err)
|
||||
|
||||
exists, err := fs.Exists("today", "touchfile.md")
|
||||
r.NoError(err)
|
||||
r.True(exists)
|
||||
}
|
||||
|
||||
func FuzzWrite(f *testing.F) {
|
||||
f.Add("valid_dir", "valid_file.txt", "This is valid content")
|
||||
f.Add("valid_dir", "../../unsafe_file.txt", "Unsafe path content")
|
||||
f.Add("valid_dir/subdir", "valid_file.md", "Nested content")
|
||||
f.Add("invalid<>|*?dir", "invalid_file.txt", "Invalid dir")
|
||||
f.Add("valid_dir", "file_with_emoji_🀀.txt", "Unicode content")
|
||||
f.Add("../", "file.md", "content")
|
||||
f.Add("../../", "file.md", "content")
|
||||
f.Add("../../../", "file.md", "content")
|
||||
f.Add("dir", "../file.md", "content")
|
||||
f.Add("dir", "../../file.md", "content")
|
||||
f.Add("dir", "../../../file.md", "content")
|
||||
|
||||
f.Fuzz(func(t *testing.T, dir, filename, content string) {
|
||||
filename = filename + ".md"
|
||||
|
||||
r := require.New(t)
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
_ = fs.Mkdir("/user", 0o755)
|
||||
|
||||
userFS, err := NewFS("/user", fs)
|
||||
r.NoError(err)
|
||||
|
||||
err = userFS.Write(dir, filename, content)
|
||||
|
||||
unsafePath := path.Join("/user/", dir, filename)
|
||||
otherUserDir := !strings.HasPrefix(unsafePath, "/user/")
|
||||
if unsafePath == "/user" {
|
||||
otherUserDir = false
|
||||
}
|
||||
if otherUserDir || strings.Contains(unsafePath, "../") {
|
||||
if !errors.Is(err, ErrUnsafePath) {
|
||||
t.Errorf("Expected unsafe path error for dir: '%s', filename: '%s', calculated path: '%s', got: '%v'", dir, filename, unsafePath, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
r.NoError(err, "Unexpected error for valid inputs dir: '%s', filename: '%s'", dir, filename)
|
||||
|
||||
filePath, err := userFS.SafePath(dir, filename)
|
||||
r.NoError(err)
|
||||
|
||||
actualContent, readErr := afero.ReadFile(userFS.backend, filePath)
|
||||
r.NoError(readErr, "Error reading file: %s", filePath)
|
||||
r.Equal(content, string(actualContent), "Content mismatch for file: %s", filePath)
|
||||
|
||||
// Check that a file is indeed created in user fs, and not outside
|
||||
files, err := userFS.FilesAndDirs("/")
|
||||
r.NoError(err, "Unexpected error for valid inputs dir: '%s', filename: '%s', calculated path: '%s", dir, filename, unsafePath)
|
||||
r.Len(files, 1, "File has been written outside of user dir. Provided dir: '%s', filename: '%s', unsafe path: '%s", dir, filename, filePath)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCtimes(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
saved := Mtime
|
||||
defer func() {
|
||||
Mtime = saved
|
||||
}()
|
||||
|
||||
Mtime = func(fi os.FileInfo) int64 {
|
||||
switch fi.Name() {
|
||||
case "file1.md":
|
||||
return 1000
|
||||
case "file2.md":
|
||||
return 2000
|
||||
case "nested.md":
|
||||
return 3000
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
fs, err := NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Write("/", "file1.md", "content1")
|
||||
r.NoError(err)
|
||||
err = fs.Write("today", "file2.md", "content2")
|
||||
r.NoError(err)
|
||||
err = fs.Write("/", "not-markdown.txt", "should be ignored")
|
||||
r.NoError(err)
|
||||
err = fs.MakeDir("today/subdir")
|
||||
r.NoError(err)
|
||||
err = fs.Write("today/subdir", "nested.md", "nested content")
|
||||
r.NoError(err)
|
||||
|
||||
// Create hidden file (should be ignored)
|
||||
err = fs.Write("today", ".hidden.md", "hidden content")
|
||||
r.NoError(err)
|
||||
|
||||
// Test Ctimes
|
||||
ctimes, err := fs.Mtimes("/", ".md")
|
||||
r.NoError(err)
|
||||
|
||||
fmt.Println(ctimes)
|
||||
r.Len(ctimes, 3)
|
||||
r.Equal(int64(1000), ctimes["file1.md"])
|
||||
r.Equal(int64(2000), ctimes["today/file2.md"])
|
||||
r.Equal(int64(3000), ctimes["today/subdir/nested.md"])
|
||||
|
||||
// Should not include non-markdown files or hidden files
|
||||
_, exists := ctimes["today/not-markdown.txt"]
|
||||
r.False(exists)
|
||||
|
||||
_, exists = ctimes["today/.hidden.md"]
|
||||
r.False(exists)
|
||||
}
|
||||
|
||||
func TestMtimesAllExtensions(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
saved := Mtime
|
||||
defer func() {
|
||||
Mtime = saved
|
||||
}()
|
||||
|
||||
Mtime = func(fi os.FileInfo) int64 {
|
||||
switch fi.Name() {
|
||||
case "file1.md":
|
||||
return 1000
|
||||
case "file2.md":
|
||||
return 2000
|
||||
case "nested.md":
|
||||
return 3000
|
||||
case "file.txt":
|
||||
return 3000
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
fs, err := NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Write("/", "file1.md", "content1")
|
||||
r.NoError(err)
|
||||
err = fs.Write("today", "file2.md", "content2")
|
||||
r.NoError(err)
|
||||
err = fs.Write("/", "file.txt", "should be ignored")
|
||||
r.NoError(err)
|
||||
err = fs.MakeDir("today/subdir")
|
||||
r.NoError(err)
|
||||
err = fs.Write("today/subdir", "nested.md", "nested content")
|
||||
r.NoError(err)
|
||||
|
||||
// Create hidden file (should be ignored)
|
||||
err = fs.Write("today", ".hidden.md", "hidden content")
|
||||
r.NoError(err)
|
||||
|
||||
ctimes, err := fs.Mtimes("/")
|
||||
r.NoError(err)
|
||||
|
||||
r.Len(ctimes, 4)
|
||||
r.Equal(int64(1000), ctimes["file1.md"])
|
||||
r.Equal(int64(2000), ctimes["today/file2.md"])
|
||||
r.Equal(int64(3000), ctimes["today/subdir/nested.md"])
|
||||
r.Equal(int64(3000), ctimes["file.txt"])
|
||||
|
||||
// Should not include non-markdown files or hidden files
|
||||
_, exists := ctimes["today/not-markdown.txt"]
|
||||
r.False(exists)
|
||||
|
||||
_, exists = ctimes["today/.hidden.md"]
|
||||
r.False(exists)
|
||||
}
|
||||
|
||||
func TestMtimesInSubDir(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
saved := Mtime
|
||||
defer func() {
|
||||
Mtime = saved
|
||||
}()
|
||||
|
||||
Mtime = func(fi os.FileInfo) int64 {
|
||||
switch fi.Name() {
|
||||
case "file1.md":
|
||||
return 1000
|
||||
case "file2.md":
|
||||
return 2000
|
||||
case "nested.md":
|
||||
return 3000
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
fs, err := NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.Write("/", "rootfile.md", "content1")
|
||||
r.NoError(err)
|
||||
err = fs.Write("today", "file1.md", "content1")
|
||||
r.NoError(err)
|
||||
err = fs.Write("today", "file2.md", "content2")
|
||||
r.NoError(err)
|
||||
err = fs.Write("today", "not-markdown.txt", "should be ignored")
|
||||
r.NoError(err)
|
||||
err = fs.MakeDir("today/subdir")
|
||||
r.NoError(err)
|
||||
err = fs.Write("today/subdir", "nested.md", "nested content")
|
||||
r.NoError(err)
|
||||
|
||||
// Create hidden file (should be ignored)
|
||||
err = fs.Write("today", ".hidden.md", "hidden content")
|
||||
r.NoError(err)
|
||||
|
||||
// Test Ctimes
|
||||
ctimes, err := fs.Mtimes("today", ".md")
|
||||
r.NoError(err)
|
||||
|
||||
// Should only include .md files, not .txt or hidden files
|
||||
r.Len(ctimes, 3)
|
||||
r.Equal(int64(1000), ctimes["file1.md"])
|
||||
r.Equal(int64(2000), ctimes["file2.md"])
|
||||
r.Equal(int64(3000), ctimes["subdir/nested.md"])
|
||||
|
||||
// Should not include non-markdown files or hidden files
|
||||
_, exists := ctimes["today/not-markdown.txt"]
|
||||
r.False(exists)
|
||||
|
||||
_, exists = ctimes["today/.hidden.md"]
|
||||
r.False(exists)
|
||||
}
|
||||
|
||||
func TestCtimesEmptyDir(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
fs, err := NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
err = fs.MakeDir("empty")
|
||||
r.NoError(err)
|
||||
|
||||
ctimes, err := fs.Mtimes("empty", ".md")
|
||||
r.NoError(err)
|
||||
r.Empty(ctimes)
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//go:build wasm
|
||||
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
var Ctime = func(fi os.FileInfo) int64 {
|
||||
if fi == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return fi.ModTime().UnixNano()
|
||||
}
|
||||
|
||||
var Mtime = func(fi os.FileInfo) int64 {
|
||||
if fi == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return fi.ModTime().UnixNano()
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//go:build windows
|
||||
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var Ctime = func(fi os.FileInfo) int64 {
|
||||
stat := fi.Sys().(*syscall.Win32FileAttributeData)
|
||||
return stat.LastAccessTime.Nanoseconds() / 1000_000_000
|
||||
}
|
||||
|
||||
var Mtime = func(fi os.FileInfo) int64 {
|
||||
stat := fi.Sys().(*syscall.Win32FileAttributeData)
|
||||
return stat.LastWriteTime.Nanoseconds() / 1000_000_000
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
var (
|
||||
storageQuotaMu sync.RWMutex
|
||||
storageQuotaCache = map[string]int64{}
|
||||
)
|
||||
|
||||
func storageUsed(rootPath string, backend afero.Fs) int64 {
|
||||
storageQuotaMu.RLock()
|
||||
used, ok := storageQuotaCache[rootPath]
|
||||
storageQuotaMu.RUnlock()
|
||||
if ok {
|
||||
return used
|
||||
}
|
||||
|
||||
var total int64
|
||||
_ = afero.Walk(backend, rootPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err == nil && !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
storageQuotaMu.Lock()
|
||||
storageQuotaCache[rootPath] = total
|
||||
storageQuotaMu.Unlock()
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
func recordQuotaUsage(rootPath string, delta int64) {
|
||||
storageQuotaMu.Lock()
|
||||
if _, ok := storageQuotaCache[rootPath]; ok {
|
||||
storageQuotaCache[rootPath] += delta
|
||||
}
|
||||
storageQuotaMu.Unlock()
|
||||
}
|
||||
|
||||
func checkQuota(rootPath string, backend afero.Fs, quotaKB int64, contentSize int64) error {
|
||||
if quotaKB <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if storageUsed(rootPath, backend)+contentSize > quotaKB*1024 {
|
||||
return ErrQuotaExceeded
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isUnlimitedQuota(userID int64, unlimitedIDs string) bool {
|
||||
if unlimitedIDs == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, idStr := range strings.Split(unlimitedIDs, ",") {
|
||||
id, err := strconv.ParseInt(strings.TrimSpace(idStr), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if id == userID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
go test fuzz v1
|
||||
string("..0")
|
||||
string("0")
|
||||
string("0")
|
||||
@@ -0,0 +1,4 @@
|
||||
go test fuzz v1
|
||||
string("0")
|
||||
string("..")
|
||||
string("0")
|
||||
@@ -0,0 +1,4 @@
|
||||
go test fuzz v1
|
||||
string("0")
|
||||
string("..0")
|
||||
string("0")
|
||||
@@ -0,0 +1,270 @@
|
||||
package habits
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rivo/uniseg"
|
||||
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
"github.com/zakirullin/files.md/server/i18n"
|
||||
"github.com/zakirullin/files.md/server/pkg/txt"
|
||||
)
|
||||
|
||||
// [1 => false, <year day> => 0, 1...]
|
||||
type Year map[int]int
|
||||
|
||||
const (
|
||||
habitSkipped = "⚪️"
|
||||
habitCompleted = "🟢"
|
||||
habitCompletedAtWeekend = "🟡"
|
||||
|
||||
MoodHabit = "Mood"
|
||||
)
|
||||
|
||||
var (
|
||||
MoodEmojis = []string{"⚪️", "🤕", "😔", "😐", "🙂", "😊"}
|
||||
errMalformedMonthLine = errors.New("malformed month line")
|
||||
now = time.Now
|
||||
)
|
||||
|
||||
// Habits returns Habit name => [day1 => 1, day2 => 0, ..., day365 => 0]
|
||||
func Habits(userFS *fs.FS, year int) (map[string]Year, error) {
|
||||
existingHabits, err := userFS.FilesAndDirs(fs.DirHabits)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("habits: can't read existing habits: %w", err)
|
||||
}
|
||||
|
||||
habits := make(map[string]Year)
|
||||
for _, existingHabit := range existingHabits {
|
||||
habits[existingHabit.DisplayName] = make(Year)
|
||||
}
|
||||
|
||||
filename := fmt.Sprintf("%d Habits.md", year)
|
||||
insightsExist, err := userFS.Exists(fs.DirInsights, filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("habits: can't check whether the file insightsExist: %w", err)
|
||||
}
|
||||
if !insightsExist {
|
||||
return habits, nil
|
||||
}
|
||||
|
||||
habitsForYearLines, err := userFS.Read(fs.DirInsights, filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("habits:read %s error: %w", filename, err)
|
||||
}
|
||||
|
||||
month := time.January
|
||||
lines := strings.Split(txt.NormNewLines(habitsForYearLines), "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Parsing month line
|
||||
isMonthLine := strings.HasPrefix(line, "###")
|
||||
if isMonthLine {
|
||||
parts := strings.Split(line, " ")
|
||||
if len(parts) < 2 {
|
||||
return nil, fmt.Errorf("read habits: can't parse month line '%s': %w", line, errMalformedMonthLine)
|
||||
}
|
||||
|
||||
// We should extract only first month:"June, some,gibberish"
|
||||
// TODO add tests
|
||||
date, err := time.Parse("January", txt.FirstWord(parts[1]))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read habits: can't parse month %s: %w", line, err)
|
||||
}
|
||||
month = date.Month()
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Tolerant reader, if we encounter gibberish,
|
||||
// we skip it. See ADRs in README.md for details for details
|
||||
// TODO preserve gibberish between parsing seesions
|
||||
daysAndHabit := strings.SplitN(line, " ", 2)
|
||||
if len(daysAndHabit) < 2 {
|
||||
continue
|
||||
}
|
||||
days, habit := daysAndHabit[0], daysAndHabit[1]
|
||||
|
||||
firstDayOfMonth := time.Date(year, month, 1, 0, 0, 0, 0, time.UTC)
|
||||
dayOfTheYear := firstDayOfMonth.YearDay()
|
||||
|
||||
// Moods line
|
||||
moodsMarker := MoodHabit
|
||||
if strings.Contains(habit, moodsMarker) {
|
||||
gr := uniseg.NewGraphemes(days)
|
||||
dayOffset := 0
|
||||
if _, ok := habits[MoodHabit]; !ok {
|
||||
habits[MoodHabit] = make(Year)
|
||||
}
|
||||
for gr.Next() {
|
||||
power := slices.Index(MoodEmojis, gr.Str())
|
||||
habits[MoodHabit][dayOfTheYear+dayOffset] = power
|
||||
dayOfTheYear++
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip gibberish
|
||||
habitsMarker := fmt.Sprintf("%s%s%s", habitSkipped, habitCompletedAtWeekend, habitCompleted)
|
||||
if !strings.ContainsAny(days, habitsMarker) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Habits line
|
||||
// [⚪️🟢... Habit name] i.e. completion status
|
||||
// for every day of the above found month
|
||||
habitName := strings.TrimSpace(habit)
|
||||
if _, ok := habits[habitName]; !ok {
|
||||
habits[habitName] = make(Year)
|
||||
}
|
||||
|
||||
// See README.md ADRs
|
||||
gr := uniseg.NewGraphemes(days)
|
||||
dayOffset := 0
|
||||
for gr.Next() {
|
||||
habits[habitName][dayOfTheYear+dayOffset] = 0
|
||||
if gr.Str() != habitSkipped {
|
||||
habits[habitName][dayOfTheYear+dayOffset] = 1
|
||||
}
|
||||
dayOfTheYear++
|
||||
}
|
||||
}
|
||||
|
||||
return habits, nil
|
||||
}
|
||||
|
||||
// LastWeekHabits returns Habit name => [day1 => 1, day2 => 0, ..., day365 => 0]
|
||||
// with the year days falling in the last week
|
||||
// FIXME doesn't work when a week falls into two years
|
||||
func LastWeekHabits(userFS *fs.FS, tz *time.Location) (map[string]Year, error) {
|
||||
habitsForYear, err := Habits(userFS, now().In(tz).Year())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("last week habits: can't get habits: %w", err)
|
||||
}
|
||||
|
||||
currentDay := now().In(tz)
|
||||
for currentDay.Weekday() != time.Monday {
|
||||
currentDay = currentDay.Add(-24 * time.Hour)
|
||||
}
|
||||
|
||||
existingHabits, err := userFS.FilesAndDirs(fs.DirHabits)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("last week habits: can't read existing habits: %w", err)
|
||||
}
|
||||
// Add default mood habit which is not in habits folder
|
||||
existingHabits = append(existingHabits, fs.File{Name: MoodHabit})
|
||||
|
||||
habits := make(map[string]Year)
|
||||
for _, habit := range existingHabits {
|
||||
habitName := strings.TrimSuffix(habit.Name, fs.MDExt)
|
||||
habits[habitName] = make(Year)
|
||||
for offset := range 7 {
|
||||
yearDay := currentDay.Add(time.Duration(offset) * 24 * time.Hour).YearDay()
|
||||
habits[habitName][yearDay] = 0
|
||||
if status, ok := habitsForYear[habitName][yearDay]; ok {
|
||||
habits[habitName][yearDay] = status
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return habits, nil
|
||||
}
|
||||
|
||||
func Write(userFS *fs.FS, year int, habits map[string]Year) error {
|
||||
habitKeys := make([]string, 0)
|
||||
for k := range habits {
|
||||
if k == MoodHabit {
|
||||
continue
|
||||
}
|
||||
habitKeys = append(habitKeys, k)
|
||||
}
|
||||
sort.Strings(habitKeys)
|
||||
if _, ok := habits[MoodHabit]; ok {
|
||||
habitKeys = append(habitKeys, MoodHabit)
|
||||
}
|
||||
|
||||
content := ""
|
||||
day := time.Date(year, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
for day.Year() < year+1 {
|
||||
habitsForMonth := ""
|
||||
for _, habitName := range habitKeys {
|
||||
statuses := ""
|
||||
dayOfMonth := day
|
||||
atLeastOneCompletion := false
|
||||
for dayOfMonth.Month() == day.Month() {
|
||||
emoji := habitSkipped
|
||||
if status, ok := habits[habitName][dayOfMonth.YearDay()]; ok {
|
||||
emoji = emojiForStatus(habitName, dayOfMonth, status)
|
||||
}
|
||||
if emoji != habitSkipped {
|
||||
atLeastOneCompletion = true
|
||||
}
|
||||
statuses += emoji
|
||||
|
||||
dayOfMonth = dayOfMonth.AddDate(0, 0, 1)
|
||||
}
|
||||
if atLeastOneCompletion {
|
||||
habitsForMonth += fmt.Sprintf("%s %s\n", statuses, habitName)
|
||||
}
|
||||
}
|
||||
|
||||
if len(habitsForMonth) != 0 {
|
||||
if len(content) > 0 {
|
||||
content += "\n"
|
||||
}
|
||||
content += fmt.Sprintf("### %s\n%s", day.Month(), habitsForMonth)
|
||||
}
|
||||
|
||||
day = day.AddDate(0, 1, 0)
|
||||
}
|
||||
|
||||
filename := fmt.Sprintf("%d Habits.md", year)
|
||||
err := userFS.Write(fs.DirInsights, filename, content)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't write habits: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func Emoji(userFS *fs.FS, habitName string) string {
|
||||
emoji, _ := userFS.Read(fs.DirHabits, fs.Filename(habitName))
|
||||
if emoji == "" {
|
||||
emoji = i18n.Emoji(habitName)
|
||||
}
|
||||
if emoji == "" {
|
||||
emoji = "⚡️"
|
||||
}
|
||||
|
||||
return emoji
|
||||
}
|
||||
|
||||
func emojiForStatus(habitName string, day time.Time, status int) string {
|
||||
if habitName == MoodHabit {
|
||||
if status < len(MoodEmojis) {
|
||||
return MoodEmojis[status]
|
||||
}
|
||||
|
||||
return habitSkipped
|
||||
}
|
||||
|
||||
if status == 1 {
|
||||
isWeekend := day.Weekday() == time.Saturday || day.Weekday() == time.Sunday
|
||||
if isWeekend {
|
||||
return habitCompletedAtWeekend
|
||||
} else {
|
||||
return habitCompleted
|
||||
}
|
||||
}
|
||||
|
||||
return habitSkipped
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package habits
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"time"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
"github.com/zakirullin/files.md/server/userconfig"
|
||||
)
|
||||
|
||||
//go:embed templates/habits.html
|
||||
var html string
|
||||
|
||||
func Render(userID int64, userFS *fs.FS) ([]byte, error) {
|
||||
tmpl, err := template.New("habits").Parse(html)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't parse habits template: %w", err)
|
||||
}
|
||||
|
||||
cfg := userconfig.NewConfig(userFS, userID, config.ServerCfg.ConfigFilename)
|
||||
|
||||
habits, err := LastWeekHabits(userFS, cfg.Timezone())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't render habit: %w", err)
|
||||
}
|
||||
|
||||
moods, ok := habits[MoodHabit]
|
||||
if ok {
|
||||
delete(habits, MoodHabit)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err = tmpl.Execute(&out, map[string]any{
|
||||
"habits": habits,
|
||||
"moods": moods,
|
||||
"moodEmojis": MoodEmojis,
|
||||
"host": config.ServerCfg.APIHost(),
|
||||
"userID": userID,
|
||||
"currentDay": time.Now().YearDay(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't render habits template: %w", err)
|
||||
}
|
||||
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package habits
|
||||
|
||||
// TODO one known bug - it won't correctly work
|
||||
// if our week falls into 2 different years
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
//go:embed testdata/month_habits.md
|
||||
var monthMD string
|
||||
|
||||
//go:embed testdata/last_month_habits.md
|
||||
var lastMonthMD string
|
||||
|
||||
//go:embed testdata/two_months_habits.md
|
||||
var twoMonthsMD string
|
||||
|
||||
func init() {
|
||||
fs.Ctime = func(fi os.FileInfo) int64 {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func TestHabits(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.CreateSystemDirs()
|
||||
r.NoError(err)
|
||||
_ = userFS.Write(fs.DirInsights, "1970 Habits.md", monthMD)
|
||||
|
||||
habits, err := Habits(userFS, 1970)
|
||||
r.NoError(err)
|
||||
|
||||
r.Len(habits, 5)
|
||||
year, ok := habits["Went to gym"]
|
||||
r.True(ok)
|
||||
|
||||
r.Len(year, 31)
|
||||
|
||||
r.EqualValues(Year{1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 1, 7: 0, 8: 0, 9: 0, 10: 0, 11: 1, 12: 0, 13: 0, 14: 1, 15: 0, 16: 0, 17: 0, 18: 1, 19: 0, 20: 1, 21: 0, 22: 0, 23: 1, 24: 0, 25: 1, 26: 0, 27: 1, 28: 0, 29: 1, 30: 0, 31: 1}, year)
|
||||
}
|
||||
|
||||
func TestHabitsForTwoMonths(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.CreateSystemDirs()
|
||||
r.NoError(err)
|
||||
_ = userFS.Write(fs.DirInsights, "1970 Habits.md", twoMonthsMD)
|
||||
|
||||
habits, err := Habits(userFS, 1970)
|
||||
r.NoError(err)
|
||||
|
||||
r.Len(habits, 2)
|
||||
year, ok := habits["Habit"]
|
||||
r.True(ok)
|
||||
|
||||
r.Len(year, 61)
|
||||
|
||||
r.EqualValues(Year{244: 0, 245: 0, 246: 0, 247: 0, 248: 0, 249: 1, 250: 0, 251: 1, 252: 0, 253: 0, 254: 0, 255: 0, 256: 1, 257: 0, 258: 0, 259: 0, 260: 1, 261: 0, 262: 0, 263: 1, 264: 1, 265: 1, 266: 0, 267: 1, 268: 1, 269: 0, 270: 1, 271: 0, 272: 1, 273: 0, 274: 0, 275: 0, 276: 1, 277: 0, 278: 0, 279: 1, 280: 1, 281: 0, 282: 1, 283: 1, 284: 0, 285: 0, 286: 0, 287: 0, 288: 0, 289: 0, 290: 0, 291: 0, 292: 1, 293: 1, 294: 0, 295: 0, 296: 1, 297: 0, 298: 1, 299: 1, 300: 1, 301: 1, 302: 1, 303: 0, 304: 1}, year)
|
||||
}
|
||||
|
||||
func TestLastMonthHabits(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.CreateSystemDirs()
|
||||
r.NoError(err)
|
||||
_ = userFS.Write(fs.DirInsights, "1970 Habits.md", lastMonthMD)
|
||||
|
||||
habits, err := Habits(userFS, 1970)
|
||||
r.NoError(err)
|
||||
|
||||
r.Len(habits, 1)
|
||||
year, ok := habits["Habit"]
|
||||
r.True(ok)
|
||||
|
||||
r.Len(year, 31)
|
||||
|
||||
completed, ok := year[335]
|
||||
r.True(ok)
|
||||
r.Equal(0, completed)
|
||||
|
||||
completed, ok = year[365]
|
||||
r.True(ok)
|
||||
r.Equal(1, completed)
|
||||
}
|
||||
|
||||
func TestLastWeekHabitsWhenWeekFallsIntoTwoMonths(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.CreateSystemDirs()
|
||||
r.NoError(err)
|
||||
_ = userFS.Write(fs.DirInsights, "1970 Habits.md", twoMonthsMD)
|
||||
_ = userFS.Write(fs.DirHabits, "Habit.md", "")
|
||||
|
||||
savedNow := now
|
||||
defer func() {
|
||||
now = savedNow
|
||||
}()
|
||||
now = func() time.Time {
|
||||
return time.Date(1970, time.September, 30, 0, 0, 0, 0, time.Local)
|
||||
}
|
||||
|
||||
habits, err := LastWeekHabits(userFS, time.UTC)
|
||||
r.NoError(err)
|
||||
r.Len(habits, 2)
|
||||
r.Len(habits["Habit"], 7)
|
||||
r.EqualValues(Year{271: 0, 272: 1, 273: 0, 274: 0, 275: 0, 276: 1, 277: 0}, habits["Habit"])
|
||||
r.EqualValues(Year{271: 5, 272: 2, 273: 5, 274: 0, 275: 5, 276: 4, 277: 0}, habits["Mood"])
|
||||
}
|
||||
|
||||
func TestLastMonthHabitsMoods(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.CreateSystemDirs()
|
||||
r.NoError(err)
|
||||
_ = userFS.Write(fs.DirInsights, "1970 Habits.md", monthMD)
|
||||
|
||||
habits, err := Habits(userFS, 1970)
|
||||
r.NoError(err)
|
||||
|
||||
year, ok := habits["Mood"]
|
||||
r.True(ok)
|
||||
|
||||
r.Len(year, 31)
|
||||
|
||||
r.EqualValues(Year{1: 5, 2: 0, 3: 3, 4: 1, 5: 0, 6: 5, 7: 5, 8: 0, 9: 0, 10: 0, 11: 5, 12: 0, 13: 5, 14: 2, 15: 4, 16: 1, 17: 0, 18: 5, 19: 0, 20: 4, 21: 0, 22: 5, 23: 0, 24: 5, 25: 4, 26: 0, 27: 5, 28: 4, 29: 0, 30: 5, 31: 0}, year)
|
||||
}
|
||||
|
||||
func TestWrite(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.CreateSystemDirs()
|
||||
r.NoError(err)
|
||||
_ = userFS.Write("insights", "2024 Habits.md", monthMD)
|
||||
|
||||
habits, err := Habits(userFS, 2024)
|
||||
r.NoError(err)
|
||||
|
||||
err = Write(userFS, 2024, habits)
|
||||
r.NoError(err)
|
||||
|
||||
updatedMonthMD, err := userFS.Read("insights", "2024 Habits.md")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal(monthMD, updatedMonthMD)
|
||||
}
|
||||
|
||||
func TestWritePreserveMoodsForPreviousMonth(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.CreateSystemDirs()
|
||||
r.NoError(err)
|
||||
_ = userFS.Write("insights", "1970 Habits.md", twoMonthsMD)
|
||||
|
||||
habits, err := Habits(userFS, 1970)
|
||||
r.NoError(err)
|
||||
|
||||
err = Write(userFS, 1970, habits)
|
||||
r.NoError(err)
|
||||
|
||||
updatedMD, err := userFS.Read("insights", "1970 Habits.md")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal(twoMonthsMD, updatedMD)
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
+2
@@ -0,0 +1,2 @@
|
||||
### December
|
||||
⚪️⚪️🟢⚪️⚪️🟡🟡⚪️🟢🟢⚪️⚪️⚪️⚪️⚪️⚪️⚪️⚪️🟢🟡⚪️⚪️🟢⚪️🟢🟢🟡🟡🟢⚪️🟢 Habit
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
### January
|
||||
⚪️⚪️🟢⚪️⚪️🟡🟡⚪️🟢🟢⚪️⚪️⚪️⚪️⚪️⚪️⚪️⚪️🟢🟡⚪️⚪️🟢⚪️🟢🟢🟡🟡🟢⚪️🟢 2 minute morning workout
|
||||
⚪️⚪️⚪️⚪️⚪️🟡⚪️🟢⚪️⚪️⚪️⚪️🟡⚪️⚪️⚪️🟢⚪️⚪️🟡🟡🟢⚪️🟢🟢⚪️🟡⚪️🟢🟢🟢 Ate consciously
|
||||
⚪️⚪️🟢⚪️⚪️🟡⚪️🟢🟢⚪️🟢⚪️⚪️⚪️⚪️⚪️🟢🟢⚪️🟡🟡🟢🟢⚪️⚪️⚪️⚪️⚪️🟢⚪️🟢 Kept posture for 2 min
|
||||
⚪️⚪️⚪️⚪️⚪️🟡⚪️⚪️⚪️⚪️🟢⚪️⚪️🟡⚪️⚪️⚪️🟢⚪️🟡⚪️⚪️🟢⚪️🟢⚪️🟡⚪️🟢⚪️🟢 Went to gym
|
||||
😊⚪️😐🤕⚪️😊😊⚪️⚪️⚪️😊⚪️😊😔🙂🤕⚪️😊⚪️🙂⚪️😊⚪️😊🙂⚪️😊🙂⚪️😊⚪️ Mood
|
||||
@@ -0,0 +1,9 @@
|
||||
Gibberish
|
||||
### January
|
||||
⚪️⚪️🟢⚪️⚪️🟡🟡⚪️🟢🟢⚪️⚪️⚪️⚪️⚪️⚪️⚪️⚪️🟢🟡⚪️⚪️🟢⚪️🟢🟢🟡🟡🟢⚪️🟢 2 minute morning workout
|
||||
⚪️⚪️⚪️⚪️⚪️🟡⚪️🟢⚪️⚪️⚪️⚪️🟡⚪️⚪️⚪️🟢⚪️⚪️🟡🟡🟢⚪️🟢🟢⚪️🟡⚪️🟢🟢🟢 Ate consciously
|
||||
⚪️⚪️🟢⚪️⚪️🟡⚪️🟢🟢⚪️🟢⚪️⚪️⚪️⚪️⚪️🟢🟢⚪️🟡🟡🟢🟢⚪️⚪️⚪️⚪️⚪️🟢⚪️🟢 Kept posture for 2 min
|
||||
⚪️⚪️⚪️⚪️⚪️🟡⚪️⚪️⚪️⚪️🟢⚪️⚪️🟡⚪️⚪️⚪️🟢⚪️🟡⚪️⚪️🟢⚪️🟢⚪️🟡⚪️🟢⚪️🟢 Went to gym
|
||||
😊⚪️😐🤕⚪️😊😊⚪️⚪️⚪️😊⚪️😊😔🙂🤕⚪️😊⚪️🙂⚪️😊⚪️😊🙂⚪️😊🙂⚪️😊⚪️ Mood
|
||||
|
||||
~Gibberish~
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
### September
|
||||
⚪️⚪️⚪️⚪️⚪️🟡⚪️🟢⚪️⚪️⚪️⚪️🟡⚪️⚪️⚪️🟢⚪️⚪️🟡🟢🟢⚪️🟢🟢⚪️🟡⚪️🟢⚪️ Habit
|
||||
🙂😐🙂😊😊😐🙂😊😊😐😊🙂🤕🤕😔🙂🤕🙂🙂😊🙂🙂⚪️😐😊😊🙂😊😔😊 Mood
|
||||
|
||||
### October
|
||||
⚪️⚪️🟡⚪️⚪️🟢🟢⚪️🟢🟡⚪️⚪️⚪️⚪️⚪️⚪️⚪️⚪️🟢🟢⚪️⚪️🟢⚪️🟡🟢🟢🟢🟢⚪️🟡 Habit
|
||||
⚪️😊🙂⚪️🙂😊🤕😊😐😐⚪️⚪️⚪️⚪️⚪️🤕🙂😐🙂⚪️🙂⚪️😊😊🙂🙂😊⚪️⚪️⚪️⚪️ Mood
|
||||
@@ -0,0 +1,63 @@
|
||||
package i18n
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var emojisByKeyword map[string]string
|
||||
|
||||
//go:embed emojis.json
|
||||
var emojisJSON string
|
||||
|
||||
func LoadEmojiFile() {
|
||||
var emojis map[string][]string
|
||||
err := json.Unmarshal([]byte(emojisJSON), &emojis)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("i18n.loadEmojiFile: can't unmarshal: %w", err))
|
||||
}
|
||||
|
||||
emojisByKeyword = make(map[string]string)
|
||||
for emoji, keywords := range emojis {
|
||||
for _, keyword := range keywords {
|
||||
emojisByKeyword[keyword] = emoji
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddEmoji adds auto emoji to a string based on keywords.
|
||||
// TODO add split to spaces etc
|
||||
func AddEmoji(str string) string {
|
||||
emoji := Emoji(str)
|
||||
if len(emoji) == 0 {
|
||||
return str
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s %s", emoji, str)
|
||||
}
|
||||
|
||||
func Emoji(str string) string {
|
||||
if len(emojisByKeyword) == 0 {
|
||||
LoadEmojiFile()
|
||||
}
|
||||
|
||||
strLower := strings.ToLower(str)
|
||||
aliases := []string{strLower, strLower + "s", strings.TrimSuffix(strLower, "s")}
|
||||
for _, alias := range aliases {
|
||||
icon := emojisByKeyword[alias]
|
||||
if icon != "" {
|
||||
return icon
|
||||
}
|
||||
}
|
||||
|
||||
for _, word := range strings.Fields(strLower) {
|
||||
icon := emojisByKeyword[word]
|
||||
if icon != "" {
|
||||
return icon
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
+12368
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"🏠️ *:completed*|:todo tasks for today:": "🏠️ Задачи на сегодня: *:completed*|:todo",
|
||||
"*:todo* left": "*:todo* осталось",
|
||||
"*:todo* tasks for today": "Задач на сегодня: *:todo*",
|
||||
"⏳ Tasks for later": "⏳ Задачи на потом",
|
||||
"🏠 Tasks for today": "🏠 Задачи на сегодня",
|
||||
"You don't have any tasks!": "У тебя нет задач!",
|
||||
"⏳ Your tasks for *later*:": "⏳ Задачи на *потом*:",
|
||||
"🏠 Home": "🏠️ Домой",
|
||||
"🔙 Back to today": "🔙 В список на сегодня",
|
||||
"Added to *:path* 👌!": "Добавил к *:path* 👌!",
|
||||
"Added to <b>:path</b> 👌!": "Добавил к <b>:path</b> 👌!",
|
||||
"📥 Saved to *:dir*. Change the dir?": "📥 Сохранил во *входящие*. Изменить папку?",
|
||||
":icon To :dir": ":icon В :dir",
|
||||
"📌 To Note": "📌 В заметку",
|
||||
"☑️ To Checklist": "☑️ В чеклист",
|
||||
"📝 To Doc": "📝 В документ",
|
||||
"📝 To File": "📝 В файл",
|
||||
"📝 Docs": "📝 Документы",
|
||||
"⬅️ Docs": "⬅️ Документы",
|
||||
"📝 Your docs:": "📝 Ваши документы:",
|
||||
"☑️ Checklists": "☑️ Чеклисты",
|
||||
"📝 Choose a doc or name a new one:": "📝 Выберите документ или отправьте мне имя для нового документа:",
|
||||
"💾 Choose a doc or name a new one:": "💾 Выберите документ или отправьте мне имя для нового документаллл:",
|
||||
"☑️ Choose a list or name a new one:": "☑️ Выберите лист или отправьте мне имя для нового списка:",
|
||||
"🗂 Choose a dir or name a new one:": "Выберите папку или отправьте мне имя для новой папки:",
|
||||
"💾 Saved to :dir!": "💾 Сохранил в :dir!",
|
||||
"📝 New file *:file* created!": "📝 Новый файл *:file* создан!",
|
||||
"🗂 New dir *:dir* created!": "🗂 Новая папка *:dir* создана!",
|
||||
"☑️ New list *:list* created!": "☑️ Новый лист *:list* создан!",
|
||||
"Got it 👌": "Запомнил 👌",
|
||||
"🏗 Your *:dir*:": "🏗 Папка *:dir*:",
|
||||
"⏳ Later": "⏳ Потом",
|
||||
"Task saved for *today*!": "Задача сохранена на *сегодня*!",
|
||||
"💾 Saved to *:path* 👌!": "💾 Сохранил в *:path* 👌!",
|
||||
"Go to Today": "Перейти в Сегодня",
|
||||
"🎉 :task completed!": "🎉 Задача \":task\" завершена!",
|
||||
"🌚 For tmrw" : "🌚️ На завтра",
|
||||
"⏳ For later": "⏳ На потом",
|
||||
"📆 For a day": "📆 На день",
|
||||
"➡️ Today": "➡️ Сегодня",
|
||||
"➡️ Move to today": "➡️ Переместить на сегодня",
|
||||
"➡️ Moved to today!": "➡️ Перемещено на сегодня!",
|
||||
"🌝️ Moved to tomorrow!": "🌝️ Перемещено на завтра!",
|
||||
"🏠 Moved to today!": "🏠️ Перемещено на сегодня!",
|
||||
"⏳ Move to later": "⏳ Переместить на потом",
|
||||
"⏳ Moved to later!": "⏳ Перемещено на потом!",
|
||||
"💚 To Journal": "💚 В журнал",
|
||||
"⬅️ Back": "⬅️ Назад",
|
||||
"✅ Complete": "✅ Завершить",
|
||||
"🗒 Notes": "🗒 Заметки",
|
||||
"🔄 <b>:count</b> recurring task(s)": "🔄 Повторяющиеся задач: <b>:count</b>",
|
||||
"📊 <b>:count</b> tasks done in total": "📊 Всего задач сделано: <b>:count</b>",
|
||||
"✨️ <b>:count</b> action(s)": "✨️ Действий: <b>:count</b>",
|
||||
"👀 Choose a dir to read": "👀 Что будем читать?",
|
||||
"Too short, minimum length: 1 character 😔!": "Длина задачи должна быть более 1 символа 😔!",
|
||||
"🔥 Tasks you've done <b>today</b>:\n\n:list":
|
||||
"🔥 Задачи, выполненные за <b>сегодня</b>:\n\n:list",
|
||||
"🎉 You've got a medal: :medal!": "🎉 Вы получили медаль: :medal!",
|
||||
"😎 You are more productive than <b>:percent%</b> of users!": "😎 Вы продуктивнее <b>:percent%</b> юзеров!",
|
||||
"📚 Read": "📚 Почитать",
|
||||
"📺 Watch": "📺 Посмотреть",
|
||||
"🌎 Places": "🌎 Места",
|
||||
"🏗 Projects": "🏗 Проекты",
|
||||
"📥 Inbox": "📥 Входящие",
|
||||
"🗓 Week": "🗓 Неделя",
|
||||
"☑️ Completed": "☑️ Завершенные",
|
||||
"🏠️ To today": "🏠️ На сегодня",
|
||||
"⏳ To later": "⏳ На потом",
|
||||
"🗑 Delete": "🗑 Удалить",
|
||||
"✉️️ *Send me the new text for:*": "✉️️ *Пришли мне новый текст задачи:*",
|
||||
"✏️ Select a task to edit:": "✏️ Выбери задачу для редактирования:",
|
||||
"🏠️ I'm done here!": "🏠️ Я всё!",
|
||||
"🦥 Select a task to postpone:": "🦥 Какую задачу отложить на потом?"
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package i18n
|
||||
|
||||
// How many refs per every constant? Maybe we can leave literal constants
|
||||
const (
|
||||
StrLater = "⏳"
|
||||
StrHome = "🏠 Home"
|
||||
StrBack = "⬅️ Back"
|
||||
StrComplete = "✅ Complete"
|
||||
StrMoveToLaterLong = "⏳ Move to later"
|
||||
StrToToday = "➡️ Move to today"
|
||||
StrToTomorrow = "🌚 To tmrw"
|
||||
StrToLater = "⏳ To later"
|
||||
StrToADay = "📆 To a day"
|
||||
StrToChecklist = "☑️ To Checklist"
|
||||
StrToFile = "📄 To File"
|
||||
StrToJournal = "💚 To Journal"
|
||||
StrToRead = "📚 To Read"
|
||||
StrToShop = "🛒 To Shop"
|
||||
StrToWatch = "📺 To Watch"
|
||||
StrGoToToday = "➡️ Today"
|
||||
StrRepeat = "🔄️ Repeat the task"
|
||||
StrQuickBtns = "⚡️ Quick buttons"
|
||||
StrMoveToBtns = "➡️ Move to buttons"
|
||||
|
||||
StrMonday = "Mon"
|
||||
StrTuesday = "Tue"
|
||||
StrWednesday = "Wed"
|
||||
StrThursday = "Thu"
|
||||
StrFriday = "Fri"
|
||||
StrSaturday = "Sat"
|
||||
StrSunday = "Sun"
|
||||
StrWeekdays = "Weekdays"
|
||||
StrEveryday = "Every day"
|
||||
)
|
||||
|
||||
var PomodoroStarted = "Pomodoro is started: you can see <code>Finished a break</code> task in your task list. Once are ready to focus on something and start working, just complete this task. It will get back in 50 minutes to let you know that you worked enough and deserved a break."
|
||||
|
||||
func Tr(str string) string {
|
||||
return str
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package journal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
"github.com/zakirullin/files.md/server/habits"
|
||||
"github.com/zakirullin/files.md/server/pkg/txt"
|
||||
)
|
||||
|
||||
var (
|
||||
Now = time.Now
|
||||
mu sync.Mutex
|
||||
userLocks map[string]*sync.Mutex
|
||||
)
|
||||
|
||||
// AddRecord adds a record for the current day.
|
||||
// Creates a file if there's no one for the current month
|
||||
func AddRecord(userFS *fs.FS, record string, timezone *time.Location) error {
|
||||
key, err := userFS.SafePath(fs.DirUserRoot, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get safe path: %w", err)
|
||||
}
|
||||
|
||||
lock := userLock(key)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
record = strings.TrimSpace(record)
|
||||
journalFilename := todayJournalFilename(timezone)
|
||||
exists, err := userFS.Exists(fs.DirJournal, journalFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var md string
|
||||
if exists {
|
||||
md, err = userFS.Read(fs.DirJournal, journalFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
md = txt.NormNewLines(md)
|
||||
md = strings.TrimSpace(md)
|
||||
if len(md) != 0 {
|
||||
md += "\n"
|
||||
}
|
||||
}
|
||||
|
||||
if !strings.Contains(md, todayHeader(timezone)) {
|
||||
md += todayHeader(timezone) + "\n"
|
||||
}
|
||||
|
||||
timestamp := Now().In(timezone).Format("`15:04`")
|
||||
if txt.HasImage(record) {
|
||||
// If there's an image - place timestamp under the image
|
||||
re := regexp.MustCompile(txt.ImgPattern)
|
||||
imgLink := re.FindString(record)
|
||||
record = strings.TrimSpace(strings.Replace(record, imgLink, "", 1))
|
||||
record = fmt.Sprintf("%s\n%s %s\n", imgLink, timestamp, strings.TrimSpace(record))
|
||||
} else {
|
||||
record = fmt.Sprintf("%s %s\n", Now().In(timezone).Format("`15:04`"), record)
|
||||
}
|
||||
|
||||
md += record
|
||||
|
||||
return userFS.Write(fs.DirJournal, journalFilename, md)
|
||||
}
|
||||
|
||||
// AddEmoji adds an emoji to the current day's record
|
||||
// Creates a file if there's no one for the current month
|
||||
func AddEmoji(userFS *fs.FS, emoji string, timezone *time.Location) error {
|
||||
if len(emoji) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
key, err := userFS.SafePath(fs.DirUserRoot, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get safe path: %w", err)
|
||||
}
|
||||
|
||||
lock := userLock(key)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
journalFilename := todayJournalFilename(timezone)
|
||||
exists, err := userFS.Exists(fs.DirJournal, journalFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !exists {
|
||||
md := fmt.Sprintf("%s %s", todayHeader(timezone), emoji)
|
||||
return userFS.Write(fs.DirJournal, journalFilename, md)
|
||||
}
|
||||
|
||||
md, err := userFS.Read(fs.DirJournal, journalFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
md = txt.NormNewLines(md)
|
||||
md = strings.TrimSpace(md)
|
||||
|
||||
todayHeaderRE := regexp.MustCompile(fmt.Sprintf(`(%s) *(.*)`, todayHeader(timezone)))
|
||||
if todayHeaderRE.MatchString(md) {
|
||||
replacement := fmt.Sprintf(`$1 ${2}%s`, emoji)
|
||||
// Prepend day's mood emoji in front of all other emojis
|
||||
if slices.Contains(habits.MoodEmojis, emoji) {
|
||||
replacement = fmt.Sprintf(`$1 %s${2}`, emoji)
|
||||
}
|
||||
md = todayHeaderRE.ReplaceAllString(md, replacement)
|
||||
} else {
|
||||
md += fmt.Sprintf("\n%s %s", todayHeader(timezone), emoji)
|
||||
}
|
||||
|
||||
err = userFS.Write(fs.DirJournal, journalFilename, md)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write to journal: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func todayJournalFilename(timezone *time.Location) string {
|
||||
return Now().In(timezone).Format("2006.01 January.md")
|
||||
}
|
||||
|
||||
func todayHeader(timezone *time.Location) string {
|
||||
nowTZ := Now().In(timezone)
|
||||
return fmt.Sprintf("## %d %s, %s", nowTZ.Day(), nowTZ.Format("January"), nowTZ.Weekday())
|
||||
}
|
||||
|
||||
func userLock(rootPath string) *sync.Mutex {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
if userLocks == nil {
|
||||
userLocks = make(map[string]*sync.Mutex)
|
||||
}
|
||||
if lock, exists := userLocks[rootPath]; exists {
|
||||
return lock
|
||||
}
|
||||
|
||||
newLock := &sync.Mutex{}
|
||||
userLocks[rootPath] = newLock
|
||||
|
||||
return newLock
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package journal
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAddRecord(t *testing.T) {
|
||||
r := require.New(t)
|
||||
savedNow := Now
|
||||
defer func() {
|
||||
Now = savedNow
|
||||
}()
|
||||
Now = func() time.Time {
|
||||
return time.Date(2023, 0o5, 30, 10, 0o4, 36, 0, time.UTC)
|
||||
}
|
||||
|
||||
type testcase struct {
|
||||
name string
|
||||
md string
|
||||
record string
|
||||
want string
|
||||
}
|
||||
|
||||
tests := []testcase{
|
||||
{
|
||||
name: "Empty MD",
|
||||
record: "note 1",
|
||||
want: "## 30 May, Tuesday\n`10:04` note 1\n",
|
||||
},
|
||||
{
|
||||
name: "No Headers",
|
||||
md: "some text",
|
||||
record: "note 1",
|
||||
want: "some text\n## 30 May, Tuesday\n`10:04` note 1\n",
|
||||
},
|
||||
{
|
||||
name: "Bare header",
|
||||
md: "## 30 May, Tuesday\n",
|
||||
record: "note 1",
|
||||
want: "## 30 May, Tuesday\n`10:04` note 1\n",
|
||||
},
|
||||
{
|
||||
name: "New daily note",
|
||||
md: "## 29 May, Tuesday\nnote 1",
|
||||
record: "note 2",
|
||||
want: "## 29 May, Tuesday\nnote 1\n## 30 May, Tuesday\n`10:04` note 2\n",
|
||||
},
|
||||
{
|
||||
name: "Append daily note",
|
||||
md: "## 29 May, Tuesday\nnote 1\n## 30 May, Tuesday\nnote 2",
|
||||
record: "note 3",
|
||||
want: "## 29 May, Tuesday\nnote 1\n## 30 May, Tuesday\nnote 2\n`10:04` note 3\n",
|
||||
},
|
||||
{
|
||||
name: "Image without caption",
|
||||
md: "some text",
|
||||
record: "",
|
||||
want: "some text\n## 30 May, Tuesday\n\n`10:04` \n",
|
||||
},
|
||||
{
|
||||
name: "Image with caption",
|
||||
md: "some text",
|
||||
record: "\nCaption",
|
||||
want: "some text\n## 30 May, Tuesday\n\n`10:04` Caption\n",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.Write(fs.DirJournal, "2023.05 May.md", test.md)
|
||||
r.NoError(err)
|
||||
|
||||
err = AddRecord(userFS, test.record, time.UTC)
|
||||
r.NoError(err)
|
||||
|
||||
md, err := userFS.Read(fs.DirJournal, "2023.05 May.md")
|
||||
r.NoError(err)
|
||||
r.Equal(test.want, md)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddEmojiNewFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
|
||||
savedNow := Now
|
||||
defer func() {
|
||||
Now = savedNow
|
||||
}()
|
||||
Now = func() time.Time {
|
||||
return time.Date(2024, time.January, 1, 1, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
err = AddEmoji(userFS, "🙂", time.UTC)
|
||||
r.NoError(err)
|
||||
|
||||
content, err := userFS.Read("journal", "2024.01 January.md")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal("## 1 January, Monday 🙂", content)
|
||||
}
|
||||
|
||||
func TestAddEmojiExistingFile(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
md := "## 0, Sunday\nSome Note\n## 1 January, Monday\nSome Note"
|
||||
err = userFS.Write("journal", "2024.01 January.md", md)
|
||||
r.NoError(err)
|
||||
|
||||
savedNow := Now
|
||||
defer func() {
|
||||
Now = savedNow
|
||||
}()
|
||||
Now = func() time.Time {
|
||||
return time.Date(2024, time.January, 1, 1, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
err = AddEmoji(userFS, "🙂", time.UTC)
|
||||
r.NoError(err)
|
||||
|
||||
content, err := userFS.Read("journal", "2024.01 January.md")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal("## 0, Sunday\nSome Note\n## 1 January, Monday 🙂\nSome Note", content)
|
||||
}
|
||||
|
||||
func TestAddEmojiExistingFileMissingDay(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
md := "## 0, Sunday\nSome Note\n"
|
||||
err = userFS.Write("journal", "2024.01 January.md", md)
|
||||
r.NoError(err)
|
||||
|
||||
savedNow := Now
|
||||
defer func() {
|
||||
Now = savedNow
|
||||
}()
|
||||
Now = func() time.Time {
|
||||
return time.Date(2024, time.January, 1, 1, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
err = AddEmoji(userFS, "🙂", time.UTC)
|
||||
r.NoError(err)
|
||||
|
||||
content, err := userFS.Read("journal", "2024.01 January.md")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal("## 0, Sunday\nSome Note\n## 1 January, Monday 🙂", content)
|
||||
}
|
||||
|
||||
func TestAddMoodEmojiExistingFileExistingEmojis(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
md := "## 0, Sunday\nSome Note\n## 1 January, Monday 🌱📵\nSome Note"
|
||||
err = userFS.Write("journal", "2024.01 January.md", md)
|
||||
r.NoError(err)
|
||||
|
||||
savedNow := Now
|
||||
defer func() {
|
||||
Now = savedNow
|
||||
}()
|
||||
Now = func() time.Time {
|
||||
return time.Date(2024, time.January, 1, 1, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
err = AddEmoji(userFS, "🙂", time.UTC)
|
||||
r.NoError(err)
|
||||
|
||||
content, err := userFS.Read("journal", "2024.01 January.md")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal("## 0, Sunday\nSome Note\n## 1 January, Monday 🙂🌱📵\nSome Note", content)
|
||||
}
|
||||
|
||||
func TestAddRegularEmojiExistingFileExistingEmojis(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
md := "## 0, Sunday\nSome Note\n## 1 January, Monday 🌱📵\nSome Note"
|
||||
err = userFS.Write("journal", "2024.01 January.md", md)
|
||||
r.NoError(err)
|
||||
|
||||
savedNow := Now
|
||||
defer func() {
|
||||
Now = savedNow
|
||||
}()
|
||||
Now = func() time.Time {
|
||||
return time.Date(2024, time.January, 1, 1, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
err = AddEmoji(userFS, "🎃", time.UTC)
|
||||
r.NoError(err)
|
||||
|
||||
content, err := userFS.Read("journal", "2024.01 January.md")
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal("## 0, Sunday\nSome Note\n## 1 January, Monday 🌱📵🎃\nSome Note", content)
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package slice
|
||||
|
||||
// Chunk splits items into chunkSize-d chunks
|
||||
// Chunk([1,2,3], 2) => [[1,2], [3]
|
||||
func Chunk[T any](items []T, chunkSize int) [][]T {
|
||||
if len(items) == 0 {
|
||||
return [][]T{}
|
||||
}
|
||||
|
||||
var chunks [][]T
|
||||
for len(items) > chunkSize {
|
||||
items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize])
|
||||
}
|
||||
|
||||
return append(chunks, items)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package slice
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSmallerChunk(t *testing.T) {
|
||||
// Test case 1: Chunk size is smaller than the length of the items slice
|
||||
items := []int{1, 2, 3, 4, 5, 6}
|
||||
chunkSize := 2
|
||||
expectedChunks := [][]int{
|
||||
{1, 2},
|
||||
{3, 4},
|
||||
{5, 6},
|
||||
}
|
||||
chunks := Chunk(items, chunkSize)
|
||||
if !reflect.DeepEqual(chunks, expectedChunks) {
|
||||
t.Errorf("Test case 1 failed. Expected chunks: %v, got: %v", expectedChunks, chunks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLargerChunk(t *testing.T) {
|
||||
r := require.New(t)
|
||||
r.Equal([][]int{{1, 2, 3, 4, 5, 6}}, Chunk([]int{1, 2, 3, 4, 5, 6}, 10))
|
||||
}
|
||||
|
||||
func TestEqualChunk(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
items := []int{1, 2, 3, 4, 5, 6}
|
||||
chunkSize := len(items)
|
||||
chunks := Chunk(items, chunkSize)
|
||||
r.Equal([][]int{{1, 2, 3, 4, 5, 6}}, chunks)
|
||||
}
|
||||
|
||||
func TestChunkEmpty(t *testing.T) {
|
||||
r := require.New(t)
|
||||
r.Equal([][]int{}, Chunk([]int{}, 2))
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
package tg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
)
|
||||
|
||||
const (
|
||||
MarkupHTML = "Html"
|
||||
)
|
||||
|
||||
// ErrFileTooBig is returned when a file exceeds Telegram's hard limit on
|
||||
// bot downloads (20MB via the hosted Bot API).
|
||||
var ErrFileTooBig = errors.New("file is too big for Telegram bots (20MB max)")
|
||||
|
||||
// TG is a simple wrapper over Telegram API.
|
||||
// It can send/edit messages. Keyboard can be attached.
|
||||
type TG struct {
|
||||
api *tgbotapi.BotAPI
|
||||
}
|
||||
|
||||
func NewTG(api *tgbotapi.BotAPI) *TG {
|
||||
return &TG{api}
|
||||
}
|
||||
|
||||
func (tg *TG) SendHTML(userID int64, text string, kb *Keyboard) (int, error) {
|
||||
return tg.Send(userID, text, kb, MarkupHTML)
|
||||
}
|
||||
|
||||
func (tg *TG) Send(userID int64, text string, kb *Keyboard, markup string) (int, error) {
|
||||
msg := tgbotapi.NewMessage(userID, text)
|
||||
msg.ReplyMarkup = tg.buildInlineKeyboard(kb)
|
||||
msg.ParseMode = markup
|
||||
|
||||
resp, err := tg.api.Send(msg)
|
||||
if err != nil {
|
||||
js, _ := json.Marshal(msg)
|
||||
return 0, fmt.Errorf("tg send: can't send json %s: %w", js, err)
|
||||
}
|
||||
|
||||
return resp.MessageID, nil
|
||||
}
|
||||
|
||||
func (tg *TG) SendImages(userID int64, photos []string) ([]int, error) {
|
||||
var photoFiles []any
|
||||
var documentFiles []any
|
||||
for _, img := range photos {
|
||||
// It seems like file_ids of images begin with "AgACA" pattern.
|
||||
// I don't want to do extra getFileMeta request.
|
||||
isPhoto := strings.HasPrefix(img, "AgACA")
|
||||
if isPhoto {
|
||||
photoFiles = append(photoFiles, tgbotapi.NewInputMediaPhoto(tgbotapi.FileID(img)))
|
||||
} else {
|
||||
documentFiles = append(documentFiles, tgbotapi.NewInputMediaDocument(tgbotapi.FileID(img)))
|
||||
}
|
||||
}
|
||||
|
||||
// Sending by groups. We send all photos as one group, and the documents as another group
|
||||
var msgIDs []int
|
||||
if len(photoFiles) > 0 {
|
||||
msg := tgbotapi.NewMediaGroup(userID, photoFiles)
|
||||
responses, err := tg.api.SendMediaGroup(msg)
|
||||
if err != nil {
|
||||
js, _ := json.Marshal(msg)
|
||||
return nil, fmt.Errorf("tg send photos: can't send json %s: %w", js, err)
|
||||
}
|
||||
for _, resp := range responses {
|
||||
msgIDs = append(msgIDs, resp.MessageID)
|
||||
}
|
||||
}
|
||||
if len(documentFiles) > 0 {
|
||||
msg := tgbotapi.NewMediaGroup(userID, documentFiles)
|
||||
responses, err := tg.api.SendMediaGroup(msg)
|
||||
if err != nil {
|
||||
js, _ := json.Marshal(msg)
|
||||
return nil, fmt.Errorf("tg send documents: can't send json %s: %w", js, err)
|
||||
}
|
||||
for _, resp := range responses {
|
||||
msgIDs = append(msgIDs, resp.MessageID)
|
||||
}
|
||||
}
|
||||
|
||||
return msgIDs, nil
|
||||
}
|
||||
|
||||
func (tg *TG) SendReaction(userID int64, msgID int, reaction string) error {
|
||||
url := fmt.Sprintf("https://api.telegram.org/bot%s/setMessageReaction", tg.api.Token)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"chat_id": userID,
|
||||
"message_id": msgID,
|
||||
"reaction": []map[string]string{
|
||||
{
|
||||
"type": "emoji",
|
||||
"emoji": reaction,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal payload: %w", err)
|
||||
}
|
||||
|
||||
resp, err := http.Post(url, "application/json", bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("telegram API error: %s", string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tg *TG) sendDocuments(userID int64, photos []string) ([]int, error) {
|
||||
var files []interface{}
|
||||
for _, img := range photos {
|
||||
files = append(files, tgbotapi.NewInputMediaDocument(tgbotapi.FileID(img)))
|
||||
}
|
||||
|
||||
msg := tgbotapi.NewMediaGroup(userID, files)
|
||||
|
||||
responses, err := tg.api.SendMediaGroup(msg)
|
||||
if err != nil {
|
||||
js, _ := json.Marshal(msg)
|
||||
return nil, fmt.Errorf("tg send photos: can't send json %s: %w", js, err)
|
||||
}
|
||||
|
||||
var msgIDs []int
|
||||
for _, resp := range responses {
|
||||
msgIDs = append(msgIDs, resp.MessageID)
|
||||
}
|
||||
|
||||
return msgIDs, nil
|
||||
}
|
||||
|
||||
func (tg *TG) Edit(userID int64, msgID int, text string, kb *Keyboard, markup string) error {
|
||||
msg := tgbotapi.NewEditMessageText(userID, msgID, text)
|
||||
|
||||
msg.ReplyMarkup = tg.buildInlineKeyboard(kb)
|
||||
msg.ParseMode = markup
|
||||
|
||||
_, err := tg.api.Send(msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tg edit: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tg *TG) Del(userID int64, msgID int) error {
|
||||
del := tgbotapi.NewDeleteMessage(userID, msgID)
|
||||
|
||||
_, err := tg.api.Send(del)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tg del: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AnswerCallbackQuery answers to incoming callbacks (keyboard's button clicks)
|
||||
func (tg *TG) AnswerCallbackQuery(queryID string, text string) error {
|
||||
_, err := tg.api.Send(tgbotapi.CallbackConfig{CallbackQueryID: queryID, Text: text})
|
||||
if err != nil {
|
||||
return fmt.Errorf("tg can't answer to callback query: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AnswerInlineQuery answers to incoming inline queries (@BotMention <text>)
|
||||
func (tg *TG) AnswerInlineQuery(queryID string, results []interface{}, cacheTime int, offset string) error {
|
||||
config := tgbotapi.InlineConfig{
|
||||
InlineQueryID: queryID,
|
||||
Results: results,
|
||||
CacheTime: cacheTime,
|
||||
NextOffset: offset,
|
||||
IsPersonal: true,
|
||||
}
|
||||
_, err := tg.api.Send(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tg can't answer to inline query: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownloadFile returns downloaded file extension
|
||||
func (tg *TG) DownloadFile(fileID string, outFile io.Writer) (string, error) {
|
||||
file, err := tg.api.GetFile(tgbotapi.FileConfig{FileID: fileID})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "file is too big") {
|
||||
return "", fmt.Errorf("tg can't get file: %w", ErrFileTooBig)
|
||||
}
|
||||
return "", fmt.Errorf("tg can't get file: %w", err)
|
||||
}
|
||||
|
||||
fileURL := fmt.Sprintf("https://api.telegram.org/file/bot%s/%s", tg.api.Token, file.FilePath)
|
||||
resp, err := http.Get(fileURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("tg can't download file: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
_, err = io.Copy(outFile, resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("tg can't save file: %w", err)
|
||||
}
|
||||
|
||||
return path.Ext(file.FilePath), nil
|
||||
}
|
||||
|
||||
func (tg *TG) ChannelCreatorID(chatID int64) (int64, error) {
|
||||
config := tgbotapi.ChatAdministratorsConfig{
|
||||
ChatConfig: tgbotapi.ChatConfig{
|
||||
ChatID: chatID,
|
||||
},
|
||||
}
|
||||
|
||||
admins, err := tg.api.GetChatAdministrators(config)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
for _, admin := range admins {
|
||||
if admin.Status == "creator" {
|
||||
return admin.User.ID, nil
|
||||
}
|
||||
}
|
||||
|
||||
return 0, fmt.Errorf("tg channel owner not found")
|
||||
}
|
||||
|
||||
func (tg *TG) buildInlineKeyboard(kb *Keyboard) *tgbotapi.InlineKeyboardMarkup {
|
||||
if kb == nil || len(kb.Btns) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
inlineKb := tgbotapi.NewInlineKeyboardMarkup()
|
||||
for _, row := range kb.Btns {
|
||||
switch typedRow := row.(type) {
|
||||
case Btn:
|
||||
btn := tg.buildBtn(typedRow)
|
||||
inlineKb.InlineKeyboard = append(inlineKb.InlineKeyboard, tgbotapi.NewInlineKeyboardRow(btn))
|
||||
case []Btn:
|
||||
inlineRow := tgbotapi.NewInlineKeyboardRow()
|
||||
for _, b := range typedRow {
|
||||
inlineRow = append(inlineRow, tg.buildBtn(b))
|
||||
}
|
||||
inlineKb.InlineKeyboard = append(inlineKb.InlineKeyboard, inlineRow)
|
||||
}
|
||||
}
|
||||
|
||||
return &inlineKb
|
||||
}
|
||||
|
||||
func (tg *TG) buildBtn(btn Btn) tgbotapi.InlineKeyboardButton {
|
||||
serializedCmd, _ := json.Marshal(btn.Cmd)
|
||||
button := tgbotapi.InlineKeyboardButton{
|
||||
Text: btn.Name,
|
||||
}
|
||||
switch btn.Cmd.Type {
|
||||
|
||||
case CmdTypeInlineQueryCurrentChat:
|
||||
{
|
||||
param := ""
|
||||
if len(btn.Cmd.Params) > 0 {
|
||||
param = btn.Cmd.Params[0]
|
||||
}
|
||||
button.SwitchInlineQueryCurrentChat = ¶m
|
||||
}
|
||||
case CmdTypeWebApp:
|
||||
{
|
||||
param := ""
|
||||
if len(btn.Cmd.Params) > 0 {
|
||||
param = btn.Cmd.Params[0]
|
||||
}
|
||||
button.WebApp = &tgbotapi.WebAppInfo{URL: param}
|
||||
}
|
||||
case CmdTypeURL:
|
||||
{
|
||||
param := ""
|
||||
if len(btn.Cmd.Params) > 0 {
|
||||
param = btn.Cmd.Params[0]
|
||||
}
|
||||
button.URL = ¶m
|
||||
}
|
||||
default:
|
||||
{
|
||||
str := string(serializedCmd)
|
||||
button.CallbackData = &str
|
||||
}
|
||||
}
|
||||
|
||||
return button
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package tg
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
type Message struct {
|
||||
Text string
|
||||
Buttons []Row
|
||||
}
|
||||
|
||||
type FakeTG struct {
|
||||
SentTexts []string
|
||||
LastSentText string
|
||||
LastEditedText string
|
||||
LastSentKeyboard *Keyboard
|
||||
LastEditedKeyboard *Keyboard
|
||||
InlineQueryResults []any
|
||||
LastSentMessageID int
|
||||
Messages []Message
|
||||
EditedMessages []Message
|
||||
}
|
||||
|
||||
func NewFakeTG() *FakeTG {
|
||||
return &FakeTG{}
|
||||
}
|
||||
|
||||
func (f *FakeTG) Send(userID int64, text string, kb *Keyboard, markup string) (int, error) {
|
||||
f.LastSentText = text
|
||||
f.SentTexts = append(f.SentTexts, text)
|
||||
f.LastSentKeyboard = kb
|
||||
f.LastEditedKeyboard = nil
|
||||
f.LastEditedText = ""
|
||||
f.LastSentMessageID++
|
||||
|
||||
msg := Message{
|
||||
Text: text,
|
||||
}
|
||||
if kb != nil {
|
||||
msg.Buttons = kb.Btns
|
||||
}
|
||||
f.Messages = append(f.Messages, msg)
|
||||
|
||||
return f.LastSentMessageID, nil
|
||||
}
|
||||
|
||||
func (tg *FakeTG) SendImages(userID int64, images []string) ([]int, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (tg *FakeTG) SendReaction(userID int64, msgID int, reaction string) error { return nil }
|
||||
|
||||
func (f *FakeTG) Edit(userID int64, msgID int, text string, kb *Keyboard, markup string) error {
|
||||
f.LastEditedText = text
|
||||
f.LastEditedKeyboard = kb
|
||||
f.LastSentKeyboard = nil
|
||||
|
||||
msg := Message{
|
||||
Text: text,
|
||||
}
|
||||
if kb != nil {
|
||||
msg.Buttons = kb.Btns
|
||||
}
|
||||
f.EditedMessages = append(f.EditedMessages, msg)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeTG) Del(userID int64, msgID int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeTG) AnswerCallbackQuery(queryID string, text string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeTG) AnswerInlineQuery(queryID string, results []interface{}, cacheTime int, offset string) error {
|
||||
f.InlineQueryResults = results
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeTG) DownloadFile(fileID string, writer io.Writer) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (f *FakeTG) ChannelCreatorID(chatID int64) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package tg
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSend(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
_, err := w.Write([]byte(`{"ok":true, "result": {"id": -2}}`))
|
||||
r.NoError(err)
|
||||
|
||||
if req.URL.Path == "/sendMessage" {
|
||||
body, err := io.ReadAll(req.Body)
|
||||
r.NoError(err)
|
||||
r.Equal("chat_id=-1&entities=null&parse_mode=Html&text=t", string(body))
|
||||
}
|
||||
}))
|
||||
|
||||
api, _ := tgbotapi.NewBotAPIWithAPIEndpoint("", ts.URL+"%s/%s")
|
||||
|
||||
tg := NewTG(api)
|
||||
|
||||
_, err := tg.SendHTML(-1, "t", nil)
|
||||
r.NoError(err)
|
||||
}
|
||||
|
||||
func TestEdit(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
_, err := w.Write([]byte(`{"ok":true, "result": {"id": -2}}`))
|
||||
r.NoError(err)
|
||||
|
||||
if req.URL.Path == "/editMessage" {
|
||||
body, err := io.ReadAll(req.Body)
|
||||
r.NoError(err)
|
||||
r.Equal("chat_id=-1&entities=null&parse_mode=Html&text=t", string(body))
|
||||
}
|
||||
}))
|
||||
|
||||
api, _ := tgbotapi.NewBotAPIWithAPIEndpoint("", ts.URL+"%s/%s")
|
||||
|
||||
tg := NewTG(api)
|
||||
|
||||
err := tg.Edit(-1, -2, "t", nil, MarkupHTML)
|
||||
r.NoError(err)
|
||||
}
|
||||
|
||||
func TestDel(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
_, err := w.Write([]byte(`{"ok":true, "result": {"id": -2}}`))
|
||||
r.NoError(err)
|
||||
|
||||
if req.URL.Path == "/deleteMessage" {
|
||||
body, err := io.ReadAll(req.Body)
|
||||
r.NoError(err)
|
||||
r.Equal("chat_id=-1&message_id=-2", string(body))
|
||||
}
|
||||
}))
|
||||
|
||||
api, _ := tgbotapi.NewBotAPIWithAPIEndpoint("", ts.URL+"%s/%s")
|
||||
|
||||
tg := NewTG(api)
|
||||
|
||||
err := tg.Del(-1, -2)
|
||||
r.NoError(err)
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
package tg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf16"
|
||||
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
)
|
||||
|
||||
var (
|
||||
errNoUserID = errors.New("update should always have at least one userID")
|
||||
errNoChannelID = errors.New("update doesn't have channelID")
|
||||
imageMimeTypes = []string{
|
||||
"image/gif",
|
||||
"image/jpeg",
|
||||
"image/pjpeg",
|
||||
"image/png",
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
}
|
||||
videoMimeTypes = []string{
|
||||
"video/mp4",
|
||||
"video/webm",
|
||||
"video/quicktime",
|
||||
}
|
||||
audioMimeTypes = []string{
|
||||
"audio/mpeg",
|
||||
"audio/mp3",
|
||||
"audio/ogg",
|
||||
"audio/wav",
|
||||
"audio/x-wav",
|
||||
"audio/webm",
|
||||
}
|
||||
)
|
||||
|
||||
// TGUpd is a simple wrapper over Telegram Update object
|
||||
type TGUpd struct {
|
||||
raw tgbotapi.Update
|
||||
}
|
||||
|
||||
func NewTGUpd(u tgbotapi.Update) *TGUpd {
|
||||
return &TGUpd{raw: u}
|
||||
}
|
||||
|
||||
func (u *TGUpd) MsgText() string {
|
||||
if u.raw.Message != nil {
|
||||
return u.raw.Message.Text
|
||||
}
|
||||
if u.raw.ChannelPost != nil {
|
||||
return u.raw.ChannelPost.Text
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (u *TGUpd) UserID() int64 {
|
||||
switch {
|
||||
case u.raw.Message != nil:
|
||||
return u.raw.Message.Chat.ID
|
||||
case u.raw.CallbackQuery != nil && u.raw.CallbackQuery.Message != nil && u.raw.CallbackQuery.Message.Chat != nil:
|
||||
return u.raw.CallbackQuery.Message.Chat.ID
|
||||
case u.raw.InlineQuery != nil && u.raw.InlineQuery.From != nil:
|
||||
return u.raw.InlineQuery.From.ID
|
||||
default:
|
||||
jsonData, _ := json.Marshal(u.raw)
|
||||
panic(fmt.Sprintf("%v: %s", errNoUserID, string(jsonData)))
|
||||
}
|
||||
}
|
||||
|
||||
func (u *TGUpd) ChannelID() (int64, bool) {
|
||||
switch {
|
||||
case u.raw.ChannelPost != nil:
|
||||
return u.raw.ChannelPost.Chat.ID, true
|
||||
case u.raw.EditedChannelPost != nil:
|
||||
return u.raw.EditedChannelPost.Chat.ID, true
|
||||
case u.raw.CallbackQuery != nil && u.raw.CallbackQuery.Message != nil && u.raw.CallbackQuery.Message.Chat != nil && u.raw.CallbackQuery.Message.Chat.Type == "channel":
|
||||
return u.raw.CallbackQuery.Message.Chat.ID, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func (u *TGUpd) ChannelName() (string, bool) {
|
||||
switch {
|
||||
case u.raw.ChannelPost != nil:
|
||||
return u.raw.ChannelPost.Chat.Title, true
|
||||
case u.raw.EditedChannelPost != nil:
|
||||
return u.raw.EditedChannelPost.Chat.Title, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func (u *TGUpd) Cmd() *Cmd {
|
||||
if u.raw.CallbackQuery != nil {
|
||||
cmd := Cmd{}
|
||||
_ = json.Unmarshal([]byte(u.raw.CallbackQuery.Data), &cmd)
|
||||
|
||||
return &cmd
|
||||
}
|
||||
|
||||
for _, entity := range u.raw.Message.Entities {
|
||||
if entity.IsCommand() {
|
||||
slashedCommand := getTextByOffset(u.raw.Message.Text, entity.Offset, entity.Length)
|
||||
|
||||
// Only return the command if the message is nothing else but the
|
||||
// command itself.
|
||||
rest := strings.TrimSpace(strings.Replace(u.raw.Message.Text, slashedCommand, "", 1))
|
||||
if rest != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
cmd := NewCmd(strings.TrimPrefix(slashedCommand, "/"), nil)
|
||||
return &cmd
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *TGUpd) MsgEntities() []tgbotapi.MessageEntity {
|
||||
if u.raw.Message != nil {
|
||||
return u.raw.Message.Entities
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *TGUpd) CaptionEntities() []tgbotapi.MessageEntity {
|
||||
if u.raw.Message != nil {
|
||||
if (u.raw.Message.CaptionEntities) != nil {
|
||||
return u.raw.Message.CaptionEntities
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *TGUpd) CallbackQueryID() (string, bool) {
|
||||
if u.raw.CallbackQuery == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return u.raw.CallbackQuery.ID, true
|
||||
}
|
||||
|
||||
func (u *TGUpd) InlineQueryID() (string, bool) {
|
||||
if u.raw.InlineQuery == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return u.raw.InlineQuery.ID, true
|
||||
}
|
||||
|
||||
func (u *TGUpd) InlineQuery() (string, bool) {
|
||||
if u.raw.InlineQuery == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return u.raw.InlineQuery.Query, true
|
||||
}
|
||||
|
||||
func (u *TGUpd) InlineQueryOffset() int {
|
||||
if u.raw.InlineQuery == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
offset, _ := strconv.Atoi(u.raw.InlineQuery.Offset)
|
||||
|
||||
return offset
|
||||
}
|
||||
|
||||
func (u *TGUpd) IsForwarded() bool {
|
||||
message := u.raw.Message
|
||||
if message == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if message.ForwardFromMessageID != 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
if message.ForwardFrom != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if message.ForwardSenderName != "" {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (u *TGUpd) IsSentViaBot() bool {
|
||||
message := u.raw.Message
|
||||
if message == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return message.ViaBot != nil
|
||||
}
|
||||
|
||||
func (u *TGUpd) ReplyToMsgID() (int, bool) {
|
||||
message := u.raw.Message
|
||||
if message == nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
if message.ReplyToMessage == nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return message.ReplyToMessage.MessageID, true
|
||||
}
|
||||
|
||||
func (u *TGUpd) PhotoOrImageID() (string, bool) {
|
||||
photoID, found := u.photoID()
|
||||
if found {
|
||||
return photoID, true
|
||||
}
|
||||
|
||||
imageID, found := u.imageID()
|
||||
if found {
|
||||
return imageID, true
|
||||
}
|
||||
|
||||
videoID, found := u.videoID()
|
||||
if found {
|
||||
return videoID, true
|
||||
}
|
||||
|
||||
audioID, found := u.audioID()
|
||||
if found {
|
||||
return audioID, true
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Caption returns the caption for the animation, audio,
|
||||
// document, paid media, photo, video or voice
|
||||
func (u *TGUpd) Caption() string {
|
||||
message := u.raw.Message
|
||||
if message == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return message.Caption
|
||||
}
|
||||
|
||||
func (u *TGUpd) photoID() (string, bool) {
|
||||
message := u.raw.Message
|
||||
if message == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if message.Photo == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Pick the photo with the maximum size, as FakeTG
|
||||
// makes some small crops
|
||||
photoSize := 0
|
||||
photoID := ""
|
||||
found := false
|
||||
for _, photo := range message.Photo {
|
||||
if photo.FileSize > photoSize {
|
||||
photoSize = photo.FileSize
|
||||
photoID = photo.FileID
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
return photoID, found
|
||||
}
|
||||
|
||||
func (u *TGUpd) imageID() (string, bool) {
|
||||
message := u.raw.Message
|
||||
if message == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if message.Document == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if slices.Contains(imageMimeTypes, message.Document.MimeType) {
|
||||
return message.Document.FileID, true
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// videoID returns the FileID of a video attached to the message - either as a
|
||||
// native Telegram video, an animation (GIF / silent mp4), or a Document with a
|
||||
// video MIME type. The bot saves the bytes via DownloadFile and the client
|
||||
// renders them inline via fold-image.js's mp4/webm/mov detection.
|
||||
func (u *TGUpd) videoID() (string, bool) {
|
||||
message := u.raw.Message
|
||||
if message == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if message.Video != nil {
|
||||
return message.Video.FileID, true
|
||||
}
|
||||
|
||||
if message.Animation != nil {
|
||||
return message.Animation.FileID, true
|
||||
}
|
||||
|
||||
if message.Document != nil && slices.Contains(videoMimeTypes, message.Document.MimeType) {
|
||||
return message.Document.FileID, true
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// audioID returns the FileID of an audio attachment - either a Telegram voice
|
||||
// message (ogg/opus), a native audio file, or a Document with an audio MIME
|
||||
// type. Same downstream path as photo/video: DownloadFile -> media/ ->
|
||||
// fold-image.js renders an <audio> element.
|
||||
func (u *TGUpd) audioID() (string, bool) {
|
||||
message := u.raw.Message
|
||||
if message == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if message.Voice != nil {
|
||||
return message.Voice.FileID, true
|
||||
}
|
||||
|
||||
if message.Audio != nil {
|
||||
return message.Audio.FileID, true
|
||||
}
|
||||
|
||||
if message.Document != nil && slices.Contains(audioMimeTypes, message.Document.MimeType) {
|
||||
return message.Document.FileID, true
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (u *TGUpd) MsgID() (int, bool) {
|
||||
if u.raw.Message != nil {
|
||||
return u.raw.Message.MessageID, true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (u *TGUpd) Time() (int, bool) {
|
||||
if u.raw.Message != nil {
|
||||
return u.raw.Message.Date, true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Takes into account Telegram's UTF-16 encoding
|
||||
// First we encode runes [128078 127997] into UTF-16 representation
|
||||
// We get string [55357 56398 55356 57341]
|
||||
// Then we decode this string back to runes
|
||||
//
|
||||
// A Unicode code point (rune) is the numerical value assigned to each character in the Unicode standard.
|
||||
// Think of runes like this: just as each letter in the alphabet has a unique position or index,
|
||||
// Unicode assigns a unique number to every character it includes, regardless of the writing system
|
||||
// or language it belongs to. This number is called the rune (code point).
|
||||
func getTextByOffset(text string, offset, length int) string {
|
||||
utfEncodedString := utf16.Encode([]rune(text))
|
||||
|
||||
runeString := utf16.Decode(utfEncodedString[offset : offset+length])
|
||||
|
||||
return string(runeString)
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
package tg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCmdNil(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
m := tgbotapi.Message{}
|
||||
m.Text = "j new journal record"
|
||||
rawUpdate := tgbotapi.Update{
|
||||
UpdateID: 0,
|
||||
Message: &m,
|
||||
}
|
||||
|
||||
u := NewTGUpd(rawUpdate)
|
||||
cmd := u.Cmd()
|
||||
|
||||
r.Nil(cmd)
|
||||
}
|
||||
|
||||
func TestCmdAlone(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
m := tgbotapi.Message{}
|
||||
m.Text = "/j"
|
||||
m.Entities = []tgbotapi.MessageEntity{{
|
||||
Type: "bot_command",
|
||||
Offset: 0,
|
||||
Length: 2,
|
||||
}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
|
||||
cmd := NewTGUpd(rawUpdate).Cmd()
|
||||
|
||||
r.NotNil(cmd)
|
||||
r.Equal("j", cmd.Name)
|
||||
}
|
||||
|
||||
// "/j " (trailing whitespace only) should still execute - the surrounding
|
||||
// space is not real content, just typing slack.
|
||||
func TestCmdAloneWithTrailingWhitespace(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
m := tgbotapi.Message{}
|
||||
m.Text = "/j "
|
||||
m.Entities = []tgbotapi.MessageEntity{{
|
||||
Type: "bot_command",
|
||||
Offset: 0,
|
||||
Length: 2,
|
||||
}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
|
||||
cmd := NewTGUpd(rawUpdate).Cmd()
|
||||
|
||||
r.NotNil(cmd)
|
||||
r.Equal("j", cmd.Name)
|
||||
}
|
||||
|
||||
// Slash-command with text after it must NOT execute - the user typed a
|
||||
// note that happens to begin with a slash, not a command invocation.
|
||||
func TestCmdAtBeginningWithText(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
m := tgbotapi.Message{}
|
||||
m.Text = "/j new journal record"
|
||||
m.Entities = []tgbotapi.MessageEntity{{
|
||||
Type: "bot_command",
|
||||
Offset: 0,
|
||||
Length: 2,
|
||||
}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
|
||||
cmd := NewTGUpd(rawUpdate).Cmd()
|
||||
|
||||
r.Nil(cmd)
|
||||
}
|
||||
|
||||
// Slash-command appearing after text must NOT execute either.
|
||||
func TestCmdAtEndWithText(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
m := tgbotapi.Message{}
|
||||
m.Text = "new journal record /j"
|
||||
m.Entities = []tgbotapi.MessageEntity{{
|
||||
Type: "bot_command",
|
||||
Offset: 19,
|
||||
Length: 2,
|
||||
}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
|
||||
cmd := NewTGUpd(rawUpdate).Cmd()
|
||||
|
||||
r.Nil(cmd)
|
||||
}
|
||||
|
||||
// The bug: a slash-command in the middle of a sentence used to be treated
|
||||
// as a command invocation. Must fall through to the text-saving path now.
|
||||
func TestCmdInTheMiddle(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
m := tgbotapi.Message{}
|
||||
m.Text = "Hey check out /habits today"
|
||||
m.Entities = []tgbotapi.MessageEntity{{
|
||||
Type: "bot_command",
|
||||
Offset: 14,
|
||||
Length: 7,
|
||||
}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
|
||||
cmd := NewTGUpd(rawUpdate).Cmd()
|
||||
|
||||
r.Nil(cmd)
|
||||
}
|
||||
|
||||
func TestUserID(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test case with a regular message
|
||||
m := tgbotapi.Message{Chat: &tgbotapi.Chat{ID: 12345}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
u := NewTGUpd(rawUpdate)
|
||||
userID := u.UserID()
|
||||
r.Equal(int64(12345), userID)
|
||||
|
||||
// Test case with a callback query
|
||||
cbq := tgbotapi.CallbackQuery{Message: &tgbotapi.Message{Chat: &tgbotapi.Chat{ID: 67890}}}
|
||||
rawUpdate = tgbotapi.Update{CallbackQuery: &cbq}
|
||||
u = NewTGUpd(rawUpdate)
|
||||
userID = u.UserID()
|
||||
r.Equal(int64(67890), userID)
|
||||
|
||||
// Test case with an inline query
|
||||
iq := tgbotapi.InlineQuery{From: &tgbotapi.User{ID: 112233}}
|
||||
rawUpdate = tgbotapi.Update{InlineQuery: &iq}
|
||||
u = NewTGUpd(rawUpdate)
|
||||
userID = u.UserID()
|
||||
r.Equal(int64(112233), userID)
|
||||
}
|
||||
|
||||
func TestMsgText(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test with a regular message
|
||||
m := tgbotapi.Message{Text: "Hello, world!"}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
u := NewTGUpd(rawUpdate)
|
||||
r.Equal("Hello, world!", u.MsgText())
|
||||
|
||||
// Test with no message
|
||||
rawUpdate = tgbotapi.Update{}
|
||||
u = NewTGUpd(rawUpdate)
|
||||
r.Equal("", u.MsgText())
|
||||
}
|
||||
|
||||
func TestIsForwarded(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test case where the message is forwarded
|
||||
m := tgbotapi.Message{ForwardFromMessageID: 1}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
u := NewTGUpd(rawUpdate)
|
||||
r.True(u.IsForwarded())
|
||||
|
||||
// Test case where the message is not forwarded
|
||||
m = tgbotapi.Message{}
|
||||
rawUpdate = tgbotapi.Update{Message: &m}
|
||||
u = NewTGUpd(rawUpdate)
|
||||
r.False(u.IsForwarded())
|
||||
}
|
||||
|
||||
func TestIsSentViaBot(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test case where the message is sent via a bot
|
||||
m := tgbotapi.Message{ViaBot: &tgbotapi.User{ID: 1}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
u := NewTGUpd(rawUpdate)
|
||||
r.True(u.IsSentViaBot())
|
||||
|
||||
// Test case where the message is not sent via a bot
|
||||
m = tgbotapi.Message{}
|
||||
rawUpdate = tgbotapi.Update{Message: &m}
|
||||
u = NewTGUpd(rawUpdate)
|
||||
r.False(u.IsSentViaBot())
|
||||
}
|
||||
|
||||
func TestReplyToMsgID(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test case where the message is a reply
|
||||
m := tgbotapi.Message{ReplyToMessage: &tgbotapi.Message{MessageID: 101}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
u := NewTGUpd(rawUpdate)
|
||||
replyID, found := u.ReplyToMsgID()
|
||||
r.True(found)
|
||||
r.Equal(101, replyID)
|
||||
|
||||
// Test case where the message is not a reply
|
||||
m = tgbotapi.Message{}
|
||||
rawUpdate = tgbotapi.Update{Message: &m}
|
||||
u = NewTGUpd(rawUpdate)
|
||||
replyID, found = u.ReplyToMsgID()
|
||||
r.False(found)
|
||||
r.Equal(0, replyID)
|
||||
}
|
||||
|
||||
func TestPhotoOrImageID(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test case with a photo
|
||||
m := tgbotapi.Message{Photo: []tgbotapi.PhotoSize{{FileID: "photo_id_123", FileSize: 200}}}
|
||||
rawUpdate := tgbotapi.Update{Message: &m}
|
||||
u := NewTGUpd(rawUpdate)
|
||||
id, found := u.PhotoOrImageID()
|
||||
r.True(found)
|
||||
r.Equal("photo_id_123", id)
|
||||
|
||||
// Test case with an image
|
||||
m = tgbotapi.Message{Document: &tgbotapi.Document{FileID: "image_id_456", MimeType: "image/png"}}
|
||||
rawUpdate = tgbotapi.Update{Message: &m}
|
||||
u = NewTGUpd(rawUpdate)
|
||||
id, found = u.PhotoOrImageID()
|
||||
r.True(found)
|
||||
r.Equal("image_id_456", id)
|
||||
|
||||
// Test case with no photo or image
|
||||
m = tgbotapi.Message{}
|
||||
rawUpdate = tgbotapi.Update{Message: &m}
|
||||
u = NewTGUpd(rawUpdate)
|
||||
id, found = u.PhotoOrImageID()
|
||||
r.False(found)
|
||||
r.Equal("", id)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package tg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
type Row interface{}
|
||||
|
||||
type Btn struct {
|
||||
Name string
|
||||
Cmd Cmd
|
||||
}
|
||||
|
||||
const (
|
||||
CmdTypeCallback = "cmd"
|
||||
CmdTypeInlineQueryCurrentChat = "iq"
|
||||
CmdTypeWebApp = "web"
|
||||
CmdTypeURL = "url"
|
||||
)
|
||||
|
||||
func NewBtn(name string, cmd Cmd) Btn {
|
||||
return Btn{name, cmd}
|
||||
}
|
||||
|
||||
// TODO remove this unnecessary method
|
||||
func NewRow(btns ...Btn) []Btn {
|
||||
return btns
|
||||
}
|
||||
|
||||
type Cmd struct {
|
||||
Name string `json:"n"`
|
||||
Params []string `json:"p"`
|
||||
Type string `json:"t"`
|
||||
}
|
||||
|
||||
func NewCmd(name string, params []string) Cmd {
|
||||
return Cmd{name, params, "cmd"}
|
||||
}
|
||||
|
||||
func NewURLCmd(url string) Cmd {
|
||||
return Cmd{"", []string{url}, CmdTypeURL}
|
||||
}
|
||||
|
||||
func NewCustomCmd(name string, params []string, cmdType string) Cmd {
|
||||
return Cmd{name, params, cmdType}
|
||||
}
|
||||
|
||||
func (c *Cmd) UnmarshalJSON(data []byte) error {
|
||||
// Unmarshal JSON to the alias
|
||||
type CmdAlias Cmd
|
||||
var ca CmdAlias
|
||||
|
||||
if err := json.Unmarshal(data, &ca); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ca.Type = "cmd"
|
||||
|
||||
*c = Cmd(ca)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Keyboard is an abstraction over Telegram's inline keyboard
|
||||
type Keyboard struct {
|
||||
Btns []Row
|
||||
}
|
||||
|
||||
// NewKeyboard accepts a slice of rows.
|
||||
// Each row is either a Btn or []Btn
|
||||
func NewKeyboard(rows []Row) *Keyboard {
|
||||
return &Keyboard{rows}
|
||||
}
|
||||
|
||||
func (k *Keyboard) AddRow(r Row) {
|
||||
k.Btns = append(k.Btns, r)
|
||||
}
|
||||
|
||||
func (k *Keyboard) PrependRow(r Row) {
|
||||
k.Btns = append([]Row{r}, k.Btns...)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMarshalCmdJSON(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
cmd := NewCmd("test", []string{"1"})
|
||||
js, err := json.Marshal(cmd)
|
||||
|
||||
r.NoError(err)
|
||||
r.Equal(`{"n":"test","p":["1"],"t":"cmd"}`, string(js))
|
||||
}
|
||||
|
||||
func TestUnmarshalCmdJSON(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
var cmd Cmd
|
||||
js := `{"n":"test","p":["1"]}`
|
||||
err := json.Unmarshal([]byte(js), &cmd)
|
||||
|
||||
r.NoError(err)
|
||||
r.Equal("cmd", cmd.Type)
|
||||
}
|
||||
|
||||
func TestRowAddBtn(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
row := NewRow()
|
||||
row = append(row, NewBtn("test", NewCmd("cmd", nil)))
|
||||
row = append(row, NewBtn("test2", NewCmd("cmd2", nil)))
|
||||
|
||||
expectedBtns := []Btn{NewBtn("test", NewCmd("cmd", nil)), NewBtn("test2", NewCmd("cmd2", nil))}
|
||||
r.Equal(expectedBtns, row)
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package tg
|
||||
|
||||
import tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
|
||||
type Upd struct {
|
||||
userID int64
|
||||
cmd Cmd
|
||||
Msg string
|
||||
PhotoID string
|
||||
PhotoCaption string
|
||||
ReplyToMessageID int
|
||||
IsSentViaBotVal bool
|
||||
InlineQueryVal string
|
||||
IsInlineQueryVal bool
|
||||
TimeVal int
|
||||
HasTimeVal bool
|
||||
}
|
||||
|
||||
func NewUpd(userID int64, msg string) *Upd {
|
||||
return &Upd{
|
||||
userID: userID,
|
||||
Msg: msg,
|
||||
ReplyToMessageID: -1,
|
||||
IsSentViaBotVal: false,
|
||||
InlineQueryVal: "",
|
||||
IsInlineQueryVal: false,
|
||||
}
|
||||
}
|
||||
|
||||
func NewUpdCmd(id int64, cmd Cmd) *Upd {
|
||||
return &Upd{userID: id, cmd: cmd}
|
||||
}
|
||||
|
||||
func (u *Upd) MsgText() string {
|
||||
return u.Msg
|
||||
}
|
||||
|
||||
func (u *Upd) UserID() int64 {
|
||||
return u.userID
|
||||
}
|
||||
|
||||
func (u *Upd) Cmd() *Cmd {
|
||||
if u.cmd.Name == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &u.cmd
|
||||
}
|
||||
|
||||
func (u *Upd) MsgEntities() []tgbotapi.MessageEntity {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *Upd) CaptionEntities() []tgbotapi.MessageEntity {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *Upd) CallbackQueryID() (string, bool) {
|
||||
return "", true
|
||||
}
|
||||
|
||||
func (u *Upd) InlineQueryID() (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (u *Upd) InlineQuery() (string, bool) {
|
||||
return u.InlineQueryVal, u.IsInlineQueryVal
|
||||
}
|
||||
|
||||
func (u *Upd) InlineQueryOffset() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (u *Upd) IsForwarded() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (u *Upd) IsSentViaBot() bool {
|
||||
return u.IsSentViaBotVal
|
||||
}
|
||||
|
||||
func (u *Upd) ReplyToMsgID() (int, bool) {
|
||||
return u.ReplyToMessageID, u.ReplyToMessageID != -1
|
||||
}
|
||||
|
||||
func (u *Upd) PhotoOrImageID() (string, bool) {
|
||||
if u.PhotoID != "" {
|
||||
return u.PhotoID, true
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (u *Upd) Caption() string {
|
||||
return u.PhotoCaption
|
||||
}
|
||||
|
||||
func (u *Upd) MsgID() (int, bool) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (u *Upd) Time() (int, bool) {
|
||||
return u.TimeVal, u.HasTimeVal
|
||||
}
|
||||
|
||||
func (u *Upd) ChannelID() (int64, bool) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (u *Upd) ChannelName() (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
package txt
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type (
|
||||
parser func(input string) []result
|
||||
result struct {
|
||||
consumed string
|
||||
left string
|
||||
}
|
||||
)
|
||||
|
||||
var openTags = map[string]string{
|
||||
"*": "<i>",
|
||||
"**": "<b>",
|
||||
"_": "<i>",
|
||||
"__": "<b>",
|
||||
}
|
||||
|
||||
// chatTimestampRE matches a leading “ `HH:MM` “ token used by chat
|
||||
// entries. Used to strip it before re-recording the body as a
|
||||
// journal/completion entry that already gets its own fresh timestamp.
|
||||
var chatTimestampRE = regexp.MustCompile("^`\\d{2}:\\d{2}` ")
|
||||
|
||||
// StripChatTimestamp drops the leading “ `HH:MM` “ token from a chat
|
||||
// entry body. Returns the input unchanged if no timestamp prefix is found.
|
||||
func StripChatTimestamp(s string) string {
|
||||
return chatTimestampRE.ReplaceAllString(s, "")
|
||||
}
|
||||
|
||||
var closeTags = map[string]string{
|
||||
"*": "</i>",
|
||||
"**": "</b>",
|
||||
"_": "</i>",
|
||||
"__": "</b>",
|
||||
}
|
||||
|
||||
func AddHeaderAndText(existingContent, header, newContent string) string {
|
||||
if !strings.Contains(existingContent, header) {
|
||||
if existingContent == "" {
|
||||
return fmt.Sprintf("%s\n%s", header, newContent)
|
||||
} else {
|
||||
return fmt.Sprintf("%s\n%s\n\n%s", header, newContent, existingContent)
|
||||
}
|
||||
}
|
||||
|
||||
lines := strings.Split(existingContent, "\n")
|
||||
headerIndex := -1
|
||||
|
||||
// Find the header line
|
||||
for i, line := range lines {
|
||||
if line == header {
|
||||
headerIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if headerIndex == -1 {
|
||||
return fmt.Sprintf("%s\n%s\n\n%s", header, newContent, existingContent)
|
||||
}
|
||||
|
||||
// Find where to insert (after the last line belonging to this header)
|
||||
insertIndex := headerIndex + 1
|
||||
|
||||
// Look for the next header or end of content
|
||||
for i := headerIndex + 1; i < len(lines); i++ {
|
||||
if strings.HasPrefix(lines[i], "###") {
|
||||
insertIndex = i
|
||||
break
|
||||
}
|
||||
insertIndex = i + 1
|
||||
}
|
||||
|
||||
// Insert the new content
|
||||
newLines := make([]string, 0, len(lines)+2)
|
||||
newLines = append(newLines, lines[:insertIndex]...)
|
||||
newLines = append(newLines, newContent)
|
||||
|
||||
// Add empty line after new content if there's content following and it's not empty
|
||||
if insertIndex < len(lines) && strings.TrimSpace(lines[insertIndex]) != "" {
|
||||
newLines = append(newLines, "")
|
||||
}
|
||||
|
||||
newLines = append(newLines, lines[insertIndex:]...)
|
||||
|
||||
return strings.Join(newLines, "\n")
|
||||
}
|
||||
|
||||
func IncompleteChecklistItems(md string) []string {
|
||||
items, isCompleted := ChecklistItems(md)
|
||||
var incomplete []string
|
||||
for _, item := range items {
|
||||
if !isCompleted[item] {
|
||||
incomplete = append(incomplete, item)
|
||||
}
|
||||
}
|
||||
|
||||
return incomplete
|
||||
}
|
||||
|
||||
func ChecklistItems(md string) ([]string, map[string]bool) {
|
||||
var items []string
|
||||
isCompleted := make(map[string]bool)
|
||||
lines := strings.Split(md, "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "- [ ] ") {
|
||||
item := strings.TrimPrefix(line, "- [ ] ")
|
||||
items = append(items, item)
|
||||
isCompleted[item] = false
|
||||
} else if strings.HasPrefix(line, "- [x] ") {
|
||||
item := strings.TrimPrefix(line, "- [x] ")
|
||||
items = append(items, item)
|
||||
isCompleted[item] = true
|
||||
}
|
||||
}
|
||||
|
||||
return items, isCompleted
|
||||
}
|
||||
|
||||
func AddChecklistItem(md, item string, checked bool) string {
|
||||
item = strings.ReplaceAll(NormNewLines(item), "\n", " ")
|
||||
|
||||
md, _ = RemoveChecklistItem(md, item)
|
||||
lines := strings.Split(md, "\n")
|
||||
|
||||
if checked {
|
||||
lines = append(lines, "- [x] "+item)
|
||||
} else {
|
||||
// Find the last incomplete item and insert before it
|
||||
insertIndex := len(lines)
|
||||
for i := len(lines) - 1; i >= 0; i-- {
|
||||
line := strings.TrimSpace(lines[i])
|
||||
if strings.HasPrefix(line, "- [ ] ") {
|
||||
insertIndex = i
|
||||
}
|
||||
}
|
||||
|
||||
// Insert the new incomplete item
|
||||
if insertIndex == len(lines) {
|
||||
lines = append(lines, "- [ ] "+item)
|
||||
} else {
|
||||
lines = append(lines[:insertIndex], append([]string{"- [ ] " + item}, lines[insertIndex:]...)...)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(strings.Join(lines, "\n"))
|
||||
}
|
||||
|
||||
// CompleteChecklistItem marks the matching item as completed in place.
|
||||
// Returns newMarkdown and modifiedItem.
|
||||
//
|
||||
// The marker stays where it was instead of being relocated to the bottom.
|
||||
// Moving it broke multi-line records on Chat.md: only the marker line
|
||||
// would migrate, leaving the continuation lines stranded above it.
|
||||
func CompleteChecklistItem(md, itemHash string) (string, string) {
|
||||
foundItem := ""
|
||||
lines := strings.Split(md, "\n")
|
||||
foundIndex := -1
|
||||
for i, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) < 6 {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasPrefix(line, "- [ ] ") && Hash(line[6:]) == itemHash {
|
||||
foundItem = line[6:]
|
||||
foundIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if foundIndex != -1 {
|
||||
lines[foundIndex] = "- [x] " + foundItem
|
||||
}
|
||||
|
||||
return strings.Join(lines, "\n"), foundItem
|
||||
}
|
||||
|
||||
// RemoveChecklistItem removes given item from checklist.
|
||||
// Returns newMarkdown, removedItem
|
||||
func RemoveChecklistItem(md, itemOrHash string) (string, string) {
|
||||
removedItem := ""
|
||||
lines := strings.Split(md, "\n")
|
||||
var newLines []string
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
// Preserve invalid lines
|
||||
if len(line) < 6 {
|
||||
newLines = append(newLines, line)
|
||||
continue
|
||||
}
|
||||
|
||||
if Hash(line[6:]) == itemOrHash || line[6:] == itemOrHash {
|
||||
removedItem = line[6:]
|
||||
continue
|
||||
}
|
||||
newLines = append(newLines, line)
|
||||
}
|
||||
return strings.Join(newLines, "\n"), removedItem
|
||||
}
|
||||
|
||||
// RemoveCompletedChecklistItems returns newMarkdown, removedMarkdown
|
||||
func RemoveCompletedChecklistItems(md string) (string, string) {
|
||||
removedMD := ""
|
||||
lines := strings.Split(md, "\n")
|
||||
var newLines []string
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) < 6 {
|
||||
newLines = append(newLines, line)
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasPrefix(line, "- [x] ") {
|
||||
removedMD += line + "\n"
|
||||
continue
|
||||
}
|
||||
newLines = append(newLines, line)
|
||||
}
|
||||
return strings.Join(newLines, "\n"), removedMD
|
||||
}
|
||||
|
||||
func ChecklistItem(md, itemOrHash string) string {
|
||||
lines := strings.Split(md, "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) < 6 {
|
||||
continue
|
||||
}
|
||||
|
||||
if Hash(line[6:]) == itemOrHash || line[6:] == itemOrHash {
|
||||
return line[6:]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// MarkdownToHTML naively converts user's markdown to Telegram-supported subset of HTML.
|
||||
// We don't need to implement full-blown AST parser because TG only supports a few HTML tags.
|
||||
// Telegram supports the following HTML tags:
|
||||
// <b>bold</b>, <strong>bold</strong>
|
||||
// <i>italic</i>, <em>italic</em>
|
||||
// <u>underline</u>, <ins>underline</ins>
|
||||
// <s>strikethrough</s>, <strike>strikethrough</strike>, <del>strikethrough</del>
|
||||
// <span class="tg-spoiler">spoiler</span>, <tg-spoiler>spoiler</tg-spoiler>
|
||||
// <b>bold <i>italic bold <s>italic bold strikethrough <span class="tg-spoiler">italic bold strikethrough spoiler</span></s> <u>underline italic bold</u></i> bold</b>
|
||||
// <a href="http://www.example.com/">inline Path</a>
|
||||
// <a href="tg://user?id=123456789">inline mention of a user</a>
|
||||
// <tg-emoji emoji-id="5368324170671202286">👍</tg-emoji>
|
||||
// <code>inline fixed-width code</code>
|
||||
// <pre>pre-formatted fixed-width code block</pre>
|
||||
// <pre><code class="language-python">pre-formatted fixed-width code block written in the Python programming language</code></pre>
|
||||
// <blockquote>Block quotation started\nBlock quotation continued\nThe last line of the block quotation</blockquote>
|
||||
// <blockquote expandable>Expandable block quotation started\nExpandable block quotation continued\nExpandable block quotation continued\nHidden by default part of the block quotation started\nExpandable block quotation continued\nThe last line of the block quotation</blockquote>
|
||||
func MarkdownToHTML(md string) string {
|
||||
mdWithoutCode := EscapeHTML(md)
|
||||
mdWithoutCode, codePlaceholders := ReplaceWithPlaceholders(mdWithoutCode, "(?s)```.*?```", "c0debl0ck")
|
||||
mdWithoutCode, inlinePlaceholders := ReplaceWithPlaceholders(mdWithoutCode, "`[^`]+`", "inl1ne")
|
||||
// By this point our markdown is safe to send as HTML via Telegram.
|
||||
// There won't be any issues like "missing closing HTML tag",
|
||||
// for the cases when our markdown has some html tags.
|
||||
// We try to convert as much markdown as possible to Telegram HTML.
|
||||
|
||||
// We split by \n\n+, because markdown context is broken by \n\n (excluding code inside ```)
|
||||
// TODO test splitting by \n\n+
|
||||
reNewLines := regexp.MustCompile(`\n{2,}`)
|
||||
segments := reNewLines.Split(mdWithoutCode, -1)
|
||||
processedSegments := make([]string, len(segments))
|
||||
for i, segment := range segments {
|
||||
// Process each segment separately
|
||||
docs := markdown()(segment)
|
||||
if len(docs) > 0 {
|
||||
segment = docs[0].consumed + docs[0].left
|
||||
}
|
||||
processedSegments[i] = segment
|
||||
}
|
||||
mdWithoutCode = strings.Join(processedSegments, "\n\n")
|
||||
|
||||
mdWithCode := RestoreFromPlaceholders(mdWithoutCode, codePlaceholders)
|
||||
mdWithCode = RestoreFromPlaceholders(mdWithCode, inlinePlaceholders)
|
||||
|
||||
// We do dirty but simple md -> html conversion.
|
||||
// Covert ` and ``` to <pre> and <code> HTML tags
|
||||
reCodeBlock := regexp.MustCompile("(?s)```(.+?)```")
|
||||
mdWithCode = reCodeBlock.ReplaceAllStringFunc(mdWithCode, func(s string) string {
|
||||
return "<pre>" + strings.TrimSpace(reCodeBlock.FindStringSubmatch(s)[1]) + "</pre>"
|
||||
})
|
||||
reInlineCode := regexp.MustCompile("`([^`]+?)`")
|
||||
mdWithCode = reInlineCode.ReplaceAllString(mdWithCode, "<code>$1</code>")
|
||||
|
||||
// Convert #+ DisplayName to <b>DisplayName</b>
|
||||
reHeader := regexp.MustCompile(`(?m)^#+\s*(.+)`)
|
||||
mdWithCode = reHeader.ReplaceAllString(mdWithCode, "<b>$1</b>")
|
||||
|
||||
return mdWithCode
|
||||
}
|
||||
|
||||
// parser Combinators. Watch an amazing video here: https://youtu.be/dDtZLm7HIJs.
|
||||
// We only support one level of nesting for bold and italic.
|
||||
func markdown() parser {
|
||||
text := notMarkdown()
|
||||
italicNoBold := or(
|
||||
and(open("*"), text, close("*")),
|
||||
and(open("_"), text, close("_")),
|
||||
)
|
||||
bold := or(
|
||||
and(open("**"), some(or(text, italicNoBold)), close("**")),
|
||||
and(open("__"), some(or(text, italicNoBold)), close("__")),
|
||||
)
|
||||
italic := or(
|
||||
and(open("*"), some(or(text, bold)), close("*")),
|
||||
and(open("_"), some(or(text, bold)), close("_")),
|
||||
)
|
||||
span := or(bold, italic, text)
|
||||
|
||||
return some(span)
|
||||
}
|
||||
|
||||
// open opens the tag
|
||||
func open(t string) parser {
|
||||
return func(input string) []result {
|
||||
if strings.HasPrefix(input, t) {
|
||||
return []result{{openTags[t], input[len(t):]}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// close closes the tag
|
||||
func close(t string) parser {
|
||||
return func(input string) []result {
|
||||
if strings.HasPrefix(input, t) {
|
||||
return []result{{closeTags[t], input[len(t):]}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// or applies multiple parsers and returns the result of the first successful parser.
|
||||
// If no parser succeeds, it returns an empty result.
|
||||
func or(parsers ...parser) parser {
|
||||
return func(input string) []result {
|
||||
var results []result
|
||||
for _, p := range parsers {
|
||||
results = append(results, p(input)...)
|
||||
}
|
||||
return results
|
||||
}
|
||||
}
|
||||
|
||||
// and applies multiple parsers in sequence.
|
||||
// Each parser must succeed in consuming part of the input.
|
||||
// If any parser fails, the whole parse fails.
|
||||
func and(parsers ...parser) parser {
|
||||
return func(input string) []result {
|
||||
results := []result{{"", input}}
|
||||
|
||||
for _, p := range parsers {
|
||||
var newResults []result
|
||||
for _, r := range results {
|
||||
for _, parsed := range p(r.left) {
|
||||
if parsed.consumed != "" {
|
||||
newResults = append(newResults, result{r.consumed + parsed.consumed, parsed.left})
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(newResults) == 0 {
|
||||
return nil
|
||||
}
|
||||
results = newResults
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
}
|
||||
|
||||
// some applies the parser one or more times. Each parse result is combined with the previous result.
|
||||
// And each parse can generate multiple results.
|
||||
func some(parser parser) parser {
|
||||
return func(input string) []result {
|
||||
return recursive(input, parser, 0)
|
||||
}
|
||||
}
|
||||
|
||||
func recursive(input string, parser parser, depth int) []result {
|
||||
var results []result
|
||||
empty := true
|
||||
for _, item := range parser(input) {
|
||||
if item.consumed == "" {
|
||||
continue
|
||||
}
|
||||
empty = false
|
||||
for _, child := range recursive(item.left, parser, depth+1) {
|
||||
results = append(results, result{item.consumed + child.consumed, child.left})
|
||||
}
|
||||
}
|
||||
if empty && depth != 0 {
|
||||
results = append(results, result{"", input})
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// notMarkdown incrementally yields when it encounters a *, **, _, __
|
||||
func notMarkdown() parser {
|
||||
return func(input string) []result {
|
||||
for i, ch := range input {
|
||||
if ch == '*' || ch == '_' {
|
||||
return []result{{input[:i], input[i:]}}
|
||||
}
|
||||
}
|
||||
if len(input) > 0 {
|
||||
return []result{{input, ""}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func Hash(filename string) string {
|
||||
hash := md5.Sum([]byte(filename))
|
||||
return hex.EncodeToString(hash[:])[:11]
|
||||
}
|
||||
@@ -0,0 +1,515 @@
|
||||
package txt
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMarkdownToHTML(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "### DisplayName\n**bold**\n*italic*"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<b>DisplayName</b>\n<b>bold</b>\n<i>italic</i>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLHeaderAndText(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "# DisplayName\nText"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<b>DisplayName</b>\nText", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLBold(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "**bold**"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<b>bold</b>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLMultilineBold(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "**bold\nstill bold**"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<b>bold\nstill bold</b>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLEmptyBoldAndItalic(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "**"
|
||||
r.Equal("**", MarkdownToHTML(md))
|
||||
md = "*"
|
||||
r.Equal("*", MarkdownToHTML(md))
|
||||
|
||||
md = "__"
|
||||
r.Equal("__", MarkdownToHTML(md))
|
||||
md = "_"
|
||||
r.Equal("_", MarkdownToHTML(md))
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLNewLineChar(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
bold := "**\n**"
|
||||
r.Equal("<b>\n</b>", MarkdownToHTML(bold))
|
||||
|
||||
italic := "*\n*"
|
||||
r.Equal("<i>\n</i>", MarkdownToHTML(italic))
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLCharAndNewLineChar(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
bold := "**a\n**"
|
||||
r.Equal("<b>a\n</b>", MarkdownToHTML(bold))
|
||||
|
||||
italic := "*a\n*"
|
||||
r.Equal("<i>a\n</i>", MarkdownToHTML(italic))
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLNewLineAndChar(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
bold := "**\na**"
|
||||
r.Equal("<b>\na</b>", MarkdownToHTML(bold))
|
||||
|
||||
italic := "*\na*"
|
||||
r.Equal("<i>\na</i>", MarkdownToHTML(italic))
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLTwoNewlinesBreakFormatting(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
bold := "**no bold\n\nno bold**"
|
||||
r.Equal("**no bold\n\nno bold**", MarkdownToHTML(bold))
|
||||
|
||||
italic := "*no italic\n\nno italic*"
|
||||
r.Equal("*no italic\n\nno italic*", MarkdownToHTML(italic))
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLMultilineBoldAndItalic(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "Some _italic text\nin two lines_, **bold text\nin two lines**, and ***bold italic text\nin two lines***."
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("Some <i>italic text\nin two lines</i>, <b>bold text\nin two lines</b>, and <b><i>bold italic text\nin two lines</i></b>.", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLHtmlInsideCode(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "```some code a > b```"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<pre>some code a > b</pre>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLInlineCodeAndCodeBlock(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "`inline code` and ```code block```"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<code>inline code</code> and <pre>code block</pre>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLInlineCodeBlockInsideCodeBlock(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "```code block with `inline code` inside```"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<pre>code block with <code>inline code</code> inside</pre>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLItalic(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "*italic*"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<i>italic</i>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLInvalidMD(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "__valid__**invalid"
|
||||
r.Equal("<b>valid</b>**invalid", MarkdownToHTML(md))
|
||||
|
||||
r.Equal("*invalid_markdown", MarkdownToHTML("*invalid_markdown"))
|
||||
r.Equal("*``invalid_markdown", MarkdownToHTML("*``invalid_markdown"))
|
||||
r.Equal("*```invalid_markdown", MarkdownToHTML("*```invalid_markdown"))
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLMultiline(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "line1\n**line2**\nline3"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("line1\n<b>line2</b>\nline3", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLBoldInsideItalic(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "*italic and __bold__*"
|
||||
r.Equal("<i>italic and <b>bold</b></i>", MarkdownToHTML(md))
|
||||
|
||||
md = "*italic and **bold***"
|
||||
r.Equal("<i>italic and <b>bold</b></i>", MarkdownToHTML(md))
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLItalicInsideBold(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "__bold and _italic___"
|
||||
r.Equal("<b>bold and <i>italic</i></b>", MarkdownToHTML(md))
|
||||
|
||||
md = "**bold and *italic***"
|
||||
r.Equal("<b>bold and <i>italic</i></b>", MarkdownToHTML(md))
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLNoLists(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "list\n1) item1\n2) item2"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("list\n1) item1\n2) item2", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLEscapeHtml(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
html := MarkdownToHTML("<a> &b")
|
||||
|
||||
r.Equal("<a> &b", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "Multiline\n# DisplayName"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("Multiline\n<b>DisplayName</b>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLMultipleHeaders(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "# Header1\n## Header2"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<b>Header1</b>\n<b>Header2</b>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLInlineCode(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "`inline code`"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<code>inline code</code>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLMultilineCodeBlock(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "```\ncode line 1\ncode line 2\n```"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<pre>code line 1\ncode line 2</pre>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLCodeWithBold(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "`code` **bold**"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<code>code</code> <b>bold</b>", html)
|
||||
}
|
||||
|
||||
func TestMarkdownToHTMLHeaderWithInlineCode(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "# DisplayName\n`inline code`"
|
||||
html := MarkdownToHTML(md)
|
||||
|
||||
r.Equal("<b>DisplayName</b>\n<code>inline code</code>", html)
|
||||
}
|
||||
|
||||
func TestChecklistItems(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] unchecked item\n- [x] checked item\n- [ ] another unchecked"
|
||||
items, isChecked := ChecklistItems(md)
|
||||
|
||||
r.Equal([]string{"unchecked item", "checked item", "another unchecked"}, items)
|
||||
|
||||
expected := map[string]bool{
|
||||
"unchecked item": false,
|
||||
"checked item": true,
|
||||
"another unchecked": false,
|
||||
}
|
||||
r.Equal(expected, isChecked)
|
||||
}
|
||||
|
||||
func TestChecklistItemsEmpty(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := ""
|
||||
items, isChecked := ChecklistItems(md)
|
||||
|
||||
r.Len(items, 0)
|
||||
r.Equal(map[string]bool{}, isChecked)
|
||||
}
|
||||
|
||||
func TestChecklistItemsWithRegularText(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "# DisplayName\n- [ ] task one\nregular text\n- [x] task two"
|
||||
items, isChecked := ChecklistItems(md)
|
||||
r.Equal([]string{"task one", "task two"}, items)
|
||||
expected := map[string]bool{
|
||||
"task one": false,
|
||||
"task two": true,
|
||||
}
|
||||
r.Equal(expected, isChecked)
|
||||
}
|
||||
|
||||
func TestChecklistItemsWithWhitespace(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := " - [ ] spaced task \n\t- [x] tabbed task\t"
|
||||
items, isChecked := ChecklistItems(md)
|
||||
r.Equal([]string{"spaced task", "tabbed task"}, items)
|
||||
|
||||
expected := map[string]bool{
|
||||
"spaced task": false,
|
||||
"tabbed task": true,
|
||||
}
|
||||
r.Equal(expected, isChecked)
|
||||
}
|
||||
|
||||
func TestAddChecklistItemUnchecked(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "existing text"
|
||||
result := AddChecklistItem(md, "new task", false)
|
||||
|
||||
r.Equal("existing text\n- [ ] new task", result)
|
||||
}
|
||||
|
||||
func TestAddChecklistItemChecked(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "existing text"
|
||||
result := AddChecklistItem(md, "completed task", true)
|
||||
|
||||
r.Equal("existing text\n- [x] completed task", result)
|
||||
}
|
||||
|
||||
func TestAddChecklistItemToEmpty(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := ""
|
||||
result := AddChecklistItem(md, "first task", false)
|
||||
|
||||
r.Equal("- [ ] first task", result)
|
||||
}
|
||||
|
||||
func TestAddChecklistItemWithNewlines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "text"
|
||||
result := AddChecklistItem(md, "task with\nnewlines", false)
|
||||
|
||||
r.Equal("text\n- [ ] task with newlines", result)
|
||||
}
|
||||
|
||||
func TestAddChecklistItemRemovesDuplicate(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] existing task\nother text"
|
||||
result := AddChecklistItem(md, "existing task", true)
|
||||
|
||||
r.Equal("other text\n- [x] existing task", result)
|
||||
}
|
||||
|
||||
func TestCompleteChecklistItem(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] task one\n- [ ] task two"
|
||||
itemHash := Hash("task one")
|
||||
result, foundItem := CompleteChecklistItem(md, itemHash)
|
||||
|
||||
r.Equal("- [x] task one\n- [ ] task two", result)
|
||||
r.Equal("task one", foundItem)
|
||||
}
|
||||
|
||||
func TestCompleteChecklistItemNotFound(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] task one\n- [ ] task two"
|
||||
result, foundItem := CompleteChecklistItem(md, "nonexistent")
|
||||
|
||||
r.Equal(md, result)
|
||||
r.Equal("", foundItem)
|
||||
}
|
||||
|
||||
func TestCompleteChecklistItemWithWhitespace(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := " - [ ] spaced task \nother text"
|
||||
itemHash := Hash("spaced task")
|
||||
result, foundItem := CompleteChecklistItem(md, itemHash)
|
||||
|
||||
r.Equal("- [x] spaced task\nother text", result)
|
||||
r.Equal("spaced task", foundItem)
|
||||
}
|
||||
|
||||
// The Chat.md flow hashes the FIRST line of a multi-line block (see
|
||||
// todayBlockHash), and CompleteChecklistItem walks the file line-by-line
|
||||
// hashing the same first-line content - so the box flips on the matched
|
||||
// line and the continuation stays right where the user wrote it.
|
||||
func TestCompleteChecklistItem_FlipsBox_WithContinuationLines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] `19:44` Five\nsix"
|
||||
result, foundItem := CompleteChecklistItem(md, Hash("`19:44` Five"))
|
||||
|
||||
r.Equal("- [x] `19:44` Five\nsix", result)
|
||||
r.Equal("`19:44` Five", foundItem)
|
||||
}
|
||||
|
||||
func TestRemoveChecklistItemByText(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] task one\n- [x] task two\n- [ ] task three"
|
||||
result, removedItem := RemoveChecklistItem(md, "task two")
|
||||
|
||||
r.Equal("- [ ] task one\n- [ ] task three", result)
|
||||
r.Equal("task two", removedItem)
|
||||
}
|
||||
|
||||
func TestRemoveChecklistItemByHash(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] task one\n- [x] task two"
|
||||
itemHash := Hash("task one")
|
||||
result, removedItem := RemoveChecklistItem(md, itemHash)
|
||||
|
||||
r.Equal("- [x] task two", result)
|
||||
r.Equal("task one", removedItem)
|
||||
}
|
||||
|
||||
func TestRemoveChecklistItemNotFound(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] task one\n- [x] task two"
|
||||
result, removedItem := RemoveChecklistItem(md, "nonexistent")
|
||||
|
||||
r.Equal(md, result)
|
||||
r.Equal("", removedItem)
|
||||
}
|
||||
|
||||
func TestRemoveChecklistItemWithRegularText(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "# DisplayName\n- [ ] task one\nregular text\n- [x] task two"
|
||||
result, removedItem := RemoveChecklistItem(md, "task one")
|
||||
|
||||
r.Equal("# DisplayName\nregular text\n- [x] task two", result)
|
||||
r.Equal("task one", removedItem)
|
||||
}
|
||||
|
||||
func TestRemoveCompletedChecklistItems(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] unchecked\n- [x] completed one\n- [ ] another unchecked\n- [x] completed two"
|
||||
result, removedMD := RemoveCompletedChecklistItems(md)
|
||||
|
||||
r.Equal("- [ ] unchecked\n- [ ] another unchecked", result)
|
||||
r.Equal("- [x] completed one\n- [x] completed two\n", removedMD)
|
||||
}
|
||||
|
||||
func TestRemoveCompletedChecklistItemsNoneCompleted(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] unchecked one\n- [ ] unchecked two"
|
||||
result, removedMD := RemoveCompletedChecklistItems(md)
|
||||
|
||||
r.Equal(md, result)
|
||||
r.Equal("", removedMD)
|
||||
}
|
||||
|
||||
func TestRemoveCompletedChecklistItemsAllCompleted(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [x] completed one\n- [x] completed two"
|
||||
result, removedMD := RemoveCompletedChecklistItems(md)
|
||||
|
||||
r.Equal("", result)
|
||||
r.Equal("- [x] completed one\n- [x] completed two\n", removedMD)
|
||||
}
|
||||
|
||||
func TestRemoveCompletedChecklistItemsWithRegularText(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "# DisplayName\n- [ ] unchecked\nregular text\n- [x] completed"
|
||||
result, removedMD := RemoveCompletedChecklistItems(md)
|
||||
|
||||
r.Equal("# DisplayName\n- [ ] unchecked\nregular text", result)
|
||||
r.Equal("- [x] completed\n", removedMD)
|
||||
}
|
||||
|
||||
func TestChecklistItem(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] task one\n- [x] task two"
|
||||
result := ChecklistItem(md, "task one")
|
||||
|
||||
r.Equal("task one", result)
|
||||
}
|
||||
|
||||
func TestChecklistItemByHash(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] task one\n- [x] task two"
|
||||
itemHash := Hash("task two")
|
||||
result := ChecklistItem(md, itemHash)
|
||||
|
||||
r.Equal("task two", result)
|
||||
}
|
||||
|
||||
func TestChecklistItemNotFound(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := "- [ ] task one\n- [x] task two"
|
||||
result := ChecklistItem(md, "nonexistent")
|
||||
|
||||
r.Equal("", result)
|
||||
}
|
||||
|
||||
func TestChecklistItemWithWhitespace(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
md := " - [ ] spaced task \nother text"
|
||||
result := ChecklistItem(md, "spaced task")
|
||||
|
||||
r.Equal("spaced task", result)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package txt
|
||||
|
||||
// Similar returns an int value in [0, 100], which stands for match level
|
||||
func Similar(str1 string, str2 string) int {
|
||||
txt1, txt2 := []rune(str1), []rune(str2)
|
||||
if len(txt1) == 0 || len(txt2) == 0 {
|
||||
return 0
|
||||
}
|
||||
return similarChar(txt1, txt2) * 200 / (len(txt1) + len(txt2))
|
||||
}
|
||||
|
||||
// returns the len of the longest string both in str1 and str2 and the positions in str1 and str2
|
||||
func similarStr(str1 []rune, str2 []rune) (int, int, int) {
|
||||
maxLen, tmp, pos1, pos2 := 0, 0, 0, 0
|
||||
len1, len2 := len(str1), len(str2)
|
||||
|
||||
for p := 0; p < len1; p++ {
|
||||
for q := 0; q < len2; q++ {
|
||||
tmp = 0
|
||||
for p+tmp < len1 && q+tmp < len2 && str1[p+tmp] == str2[q+tmp] {
|
||||
tmp++
|
||||
}
|
||||
if tmp > maxLen {
|
||||
maxLen, pos1, pos2 = tmp, p, q
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return maxLen, pos1, pos2
|
||||
}
|
||||
|
||||
// returns the total length of longest string both in str1 and str2
|
||||
func similarChar(str1 []rune, str2 []rune) int {
|
||||
maxLen, pos1, pos2 := similarStr(str1, str2)
|
||||
total := maxLen
|
||||
|
||||
if maxLen != 0 {
|
||||
if pos1 > 0 && pos2 > 0 {
|
||||
total += similarChar(str1[:pos1], str2[:pos2])
|
||||
}
|
||||
if pos1+maxLen < len(str1) && pos2+maxLen < len(str2) {
|
||||
total += similarChar(str1[pos1+maxLen:], str2[pos2+maxLen:])
|
||||
}
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package txt
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
func I64(i int64) string {
|
||||
return strconv.FormatInt(i, 10)
|
||||
}
|
||||
|
||||
func Ucfirst(str string) string {
|
||||
if len(str) == 0 {
|
||||
return str
|
||||
}
|
||||
r := []rune(str)
|
||||
|
||||
return string(unicode.ToUpper(r[0])) + string(r[1:])
|
||||
}
|
||||
|
||||
func Lcfirst(str string) string {
|
||||
if len(str) == 0 {
|
||||
return str
|
||||
}
|
||||
r := []rune(str)
|
||||
|
||||
return string(unicode.ToLower(r[0])) + string(r[1:])
|
||||
}
|
||||
|
||||
// Substr respects unicode codepoints, but not multi-unicode-codepoint aware.
|
||||
// Specifying skintone or gender of an emoji will count as 2 codepoints:
|
||||
// https://unicode.org/emoji/charts/full-emoji-modifiers.html
|
||||
func Substr(input string, start int, length int) string {
|
||||
if start < 0 || length < 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
asRunes := []rune(input)
|
||||
if start >= len(asRunes) {
|
||||
return ""
|
||||
}
|
||||
|
||||
if start+length > len(asRunes) {
|
||||
length = len(asRunes) - start
|
||||
}
|
||||
|
||||
return string(asRunes[start : start+length])
|
||||
}
|
||||
|
||||
func Emoji(emoji, str string) string {
|
||||
if emoji == "" {
|
||||
return str
|
||||
}
|
||||
|
||||
// Custom for George :)
|
||||
str = strings.TrimPrefix(str, "WRK ")
|
||||
str = strings.TrimPrefix(str, "UA ")
|
||||
str = strings.TrimPrefix(str, "US ")
|
||||
str = strings.TrimPrefix(str, "CY ")
|
||||
str = strings.TrimPrefix(str, "HOB ")
|
||||
str = strings.TrimPrefix(str, "SRB ")
|
||||
str = strings.TrimPrefix(str, "PL ")
|
||||
|
||||
return fmt.Sprintf("%s %s", emoji, str)
|
||||
}
|
||||
|
||||
func NormNewLines(text string) string {
|
||||
text = strings.Replace(text, "\r\n", "\n", -1) // replace Windows line endings
|
||||
return strings.Replace(text, "\r", "\n", -1) // replace remaining Mac line endings
|
||||
}
|
||||
|
||||
func IsMultiline(text string) bool {
|
||||
text = NormNewLines(text)
|
||||
lines := strings.Split(text, "\n")
|
||||
return len(lines) > 1
|
||||
}
|
||||
|
||||
// SplitTextIntoChunks splits the text into chunks less than or equal to maxLen.
|
||||
// The chunks are split at the last new line or space before maxLen (inclusive).
|
||||
// Spaces-like characters are trimmed out from the beginning and the end of each chunk.
|
||||
func SplitTextIntoChunks(text string, maxLen int) []string {
|
||||
text = strings.TrimSpace(text)
|
||||
|
||||
if maxLen <= 0 {
|
||||
return []string{text}
|
||||
}
|
||||
|
||||
var chunks []string
|
||||
runes := []rune(text) // Convert the string to runes
|
||||
|
||||
for len(runes) > maxLen {
|
||||
// Find the split index
|
||||
splitIndex := -1
|
||||
subStr := runes[:maxLen]
|
||||
// Find the last newline in the substring
|
||||
for i := len(subStr) - 1; i >= 0; i-- {
|
||||
if subStr[i] == '\n' {
|
||||
splitIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if splitIndex == -1 {
|
||||
// No newline found, find the last space
|
||||
for i := len(subStr) - 1; i >= 0; i-- {
|
||||
if subStr[i] == ' ' {
|
||||
splitIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if splitIndex == -1 {
|
||||
// No space found either, split at maxLen
|
||||
splitIndex = maxLen
|
||||
}
|
||||
|
||||
trimmedSubStr := strings.TrimSpace(string(runes[:splitIndex]))
|
||||
if len(trimmedSubStr) > 0 {
|
||||
chunks = append(chunks, trimmedSubStr)
|
||||
}
|
||||
runes = runes[splitIndex:]
|
||||
runes = []rune(strings.TrimSpace(string(runes)))
|
||||
}
|
||||
|
||||
// Add the remaining runes as the final chunk
|
||||
chunks = append(chunks, strings.TrimSpace(string(runes)))
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
func FirstWord(str string) string {
|
||||
str = strings.TrimSpace(str)
|
||||
re := regexp.MustCompile(`^[^\s\p{P}]+`)
|
||||
return re.FindString(str)
|
||||
}
|
||||
|
||||
func EscapeHTML(str string) string {
|
||||
// HTML escaping
|
||||
htmlEscaper := strings.NewReplacer(
|
||||
"&", "&",
|
||||
"<", "<",
|
||||
">", ">",
|
||||
)
|
||||
|
||||
return htmlEscaper.Replace(str)
|
||||
}
|
||||
|
||||
func StripHTMLTags(str string) string {
|
||||
re := regexp.MustCompile(`<[^>]*>`)
|
||||
return re.ReplaceAllString(str, "")
|
||||
}
|
||||
|
||||
func ReplaceWithPlaceholders(str, regex, placeholder string) (string, map[string]string) {
|
||||
re := regexp.MustCompile(regex)
|
||||
placeholders := make(map[string]string)
|
||||
counter := 0
|
||||
|
||||
// Function to replace each match with a placeholder
|
||||
res := re.ReplaceAllStringFunc(str, func(match string) string {
|
||||
p := fmt.Sprintf("#%s%d#", placeholder, counter)
|
||||
placeholders[p] = match
|
||||
counter++
|
||||
return p
|
||||
})
|
||||
|
||||
return res, placeholders
|
||||
}
|
||||
|
||||
func RestoreFromPlaceholders(str string, placeholders map[string]string) string {
|
||||
for placeholder, original := range placeholders {
|
||||
str = strings.ReplaceAll(str, placeholder, original)
|
||||
}
|
||||
return str
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
package txt
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPositiveI64ToStr(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
s := I64(1)
|
||||
|
||||
r.Equal("1", s)
|
||||
}
|
||||
|
||||
func TestNegativeI64ToStr(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
s := I64(-1)
|
||||
|
||||
r.Equal("-1", s)
|
||||
}
|
||||
|
||||
func TestZeroI64ToStr(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
s := I64(0)
|
||||
|
||||
r.Equal("0", s)
|
||||
}
|
||||
|
||||
func TestUcfirst(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
res := Ucfirst("abc")
|
||||
|
||||
r.Equal("Abc", res)
|
||||
}
|
||||
|
||||
func TestUcfirstRu(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
res := Ucfirst("абв")
|
||||
|
||||
r.Equal("Абв", res)
|
||||
}
|
||||
|
||||
func TestLcfirst(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
res := Lcfirst("ABC")
|
||||
|
||||
r.Equal("aBC", res)
|
||||
}
|
||||
|
||||
func TestLcfirstRu(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
res := Lcfirst("АБВ")
|
||||
|
||||
r.Equal("аБВ", res)
|
||||
}
|
||||
|
||||
func TestInsertExtAfterHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
content := AddHeaderAndText("#### 1 January 1970, Thursday\nExisting\ncontent", "#### 1 January 1970, Thursday", "New\ncontent")
|
||||
r.Equal("#### 1 January 1970, Thursday\nExisting\ncontent\nNew\ncontent", content)
|
||||
}
|
||||
|
||||
func TestInsertTextAfterHeaderNoHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := AddHeaderAndText("### header 1\nitem1\nitem2", "### header 5", "new item")
|
||||
r.Equal("### header 5\nnew item\n\n### header 1\nitem1\nitem2", content)
|
||||
}
|
||||
|
||||
func TestInsertTextAfterHeaderAtEnd(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := AddHeaderAndText("### header 1\nitem1\nitem2\n### header 2", "### header 1", "new item")
|
||||
r.Equal("### header 1\nitem1\nitem2\nnew item\n\n### header 2", content)
|
||||
}
|
||||
|
||||
func TestInsertTextAfterHeaderInMiddle(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := AddHeaderAndText("### header 0\n### header 1\nitem1\nitem2\n### header 2", "### header 1", "new item")
|
||||
r.Equal("### header 0\n### header 1\nitem1\nitem2\nnew item\n\n### header 2", content)
|
||||
}
|
||||
|
||||
func TestInsertTextAfterHeaderWithOnlyHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := AddHeaderAndText("### header 0\n### header 1\n### header 2", "### header 1", "new item")
|
||||
r.Equal("### header 0\n### header 1\nnew item\n\n### header 2", content)
|
||||
}
|
||||
|
||||
func TestInsertTextAfterHeaderAtVeryEnd(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := AddHeaderAndText("### header 1\nitem1\nitem2", "### header 1", "new item")
|
||||
r.Equal("### header 1\nitem1\nitem2\nnew item", content)
|
||||
}
|
||||
|
||||
func TestInsertTextAfterNoHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
content := AddHeaderAndText("item1\nitem2", "### header 1", "new item")
|
||||
r.Equal("### header 1\nnew item\n\nitem1\nitem2", content)
|
||||
}
|
||||
|
||||
func TestSplitTextIntoChunks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
maxLen int
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "basic split with spaces",
|
||||
input: "This is a test to check the splitting of text",
|
||||
maxLen: 10,
|
||||
expected: []string{"This is a", "test to", "check the", "splitting", "of text"},
|
||||
},
|
||||
{
|
||||
name: "split with newlines",
|
||||
input: "Line one\nLine two\nLine three",
|
||||
maxLen: 15,
|
||||
expected: []string{"Line one", "Line two", "Line three"},
|
||||
},
|
||||
{
|
||||
name: "long string without spaces",
|
||||
input: "supercalifragilisticexpialidocious",
|
||||
maxLen: 10,
|
||||
expected: []string{"supercalif", "ragilistic", "expialidoc", "ious"},
|
||||
},
|
||||
{
|
||||
name: "exact match",
|
||||
input: "ExactMatch",
|
||||
maxLen: 10,
|
||||
expected: []string{"ExactMatch"},
|
||||
},
|
||||
{
|
||||
name: "trailing and leading spaces",
|
||||
input: " Leading and trailing spaces ",
|
||||
maxLen: 15,
|
||||
expected: []string{"Leading and", "trailing spaces"},
|
||||
},
|
||||
{
|
||||
name: "no split needed",
|
||||
input: "Short text",
|
||||
maxLen: 50,
|
||||
expected: []string{"Short text"},
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
maxLen: 10,
|
||||
expected: []string{""},
|
||||
},
|
||||
{
|
||||
name: "string with only spaces",
|
||||
input: " ",
|
||||
maxLen: 10,
|
||||
expected: []string{""},
|
||||
},
|
||||
{
|
||||
name: "string with only spaces",
|
||||
input: "aaa",
|
||||
maxLen: 2,
|
||||
expected: []string{"aa", "a"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
res := SplitTextIntoChunks(tt.input, tt.maxLen)
|
||||
require.Equal(t, tt.expected, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormNewLines(t *testing.T) {
|
||||
// Initialize the test cases
|
||||
testCases := []struct {
|
||||
input string
|
||||
expectedOutput string
|
||||
}{
|
||||
// Test Case 1: Windows-style line endings
|
||||
{input: "Hello\r\nWorld", expectedOutput: "Hello\nWorld"},
|
||||
|
||||
// Test Case 2: Mac OS line endings
|
||||
{input: "Line1\rLine2\nLine3", expectedOutput: "Line1\nLine2\nLine3"},
|
||||
|
||||
// Test Case 3: Unix-style line endings (no change expected)
|
||||
{input: "This\nis\na\ntest", expectedOutput: "This\nis\na\ntest"},
|
||||
|
||||
// Test Case 4: No line endings
|
||||
{input: "NoLineEndingsHere", expectedOutput: "NoLineEndingsHere"},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
output := NormNewLines(tc.input)
|
||||
require.Equal(t, tc.expectedOutput, output, "Input: %s", tc.input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmoji(t *testing.T) {
|
||||
// Test with a valid emoji
|
||||
res := Emoji("😊", "Hello")
|
||||
require.Equal(t, "😊 Hello", res)
|
||||
|
||||
// Test with an empty emoji (should just return the string)
|
||||
res = Emoji("", "Hello")
|
||||
require.Equal(t, "Hello", res)
|
||||
|
||||
// Test with a different emoji
|
||||
res = Emoji("🎉", "Congratulations")
|
||||
require.Equal(t, "🎉 Congratulations", res)
|
||||
|
||||
// Test with an empty string for str
|
||||
res = Emoji("🎉", "")
|
||||
require.Equal(t, "🎉 ", res)
|
||||
|
||||
// Test with both emoji and str empty
|
||||
res = Emoji("", "")
|
||||
require.Equal(t, "", res)
|
||||
}
|
||||
|
||||
func TestSubstr(t *testing.T) {
|
||||
// Basic test case
|
||||
require.Equal(t, "ello", Substr("hello", 1, 4), "Should return 'ello'")
|
||||
|
||||
// Test start index beyond string length
|
||||
require.Equal(t, "", Substr("hello", 10, 2), "Should return empty string for out-of-bound start index")
|
||||
|
||||
// Test length exceeding string length
|
||||
require.Equal(t, "hello", Substr("hello", 0, 10), "Should return entire string when length exceeds")
|
||||
|
||||
// Test empty string
|
||||
require.Equal(t, "", Substr("", 0, 1), "Should return empty string when input is empty")
|
||||
|
||||
// Test Unicode characters (basic)
|
||||
require.Equal(t, "界", Substr("世界", 1, 1), "Should return '界' for unicode characters")
|
||||
|
||||
// Test handling of emoji (simple)
|
||||
require.Equal(t, "👋", Substr("👋🌍", 0, 1), "Should return first emoji '👋'")
|
||||
|
||||
// Test handling of emoji with skin tone (skin tone modifier counts as 2 codepoints)
|
||||
require.Equal(t, "👋🏻", Substr("👋🏻🌍", 0, 2), "Should return first emoji with skin tone modifier '👋🏻'")
|
||||
|
||||
// Test for slicing emoji across boundaries (shouldn't break)
|
||||
require.Equal(t, "👋🏻", Substr("👋🏻🌍", 0, 2), "Should return the entire emoji with modifier '👋🏻'")
|
||||
|
||||
// Test when slicing exceeds bounds with emoji and unicode characters
|
||||
require.Equal(t, "👋🏻🌍", Substr("👋🏻🌍", 0, 5), "Should return the whole string with emoji and modifier")
|
||||
|
||||
// Test invalid range where length is 0
|
||||
require.Equal(t, "", Substr("hello", 0, 0), "Should return an empty string when length is 0")
|
||||
}
|
||||
|
||||
func TestFirstWord(t *testing.T) {
|
||||
// Basic test case
|
||||
require.Equal(t, "Hello", FirstWord("Hello world!"), "Should return 'Hello' as the first word")
|
||||
|
||||
// Test with leading spaces
|
||||
require.Equal(t, "Hello", FirstWord(" Hello world!"), "Should trim leading spaces and return 'Hello'")
|
||||
|
||||
// Test with trailing spaces
|
||||
require.Equal(t, "Hello", FirstWord("Hello world! "), "Should trim trailing spaces and return 'Hello'")
|
||||
|
||||
// Test with punctuation
|
||||
require.Equal(t, "Hello", FirstWord("Hello, world!"), "Should return 'Hello' ignoring punctuation")
|
||||
|
||||
// Test empty string
|
||||
require.Equal(t, "", FirstWord(""), "Should return an empty string for an empty input")
|
||||
|
||||
// Test with only spaces
|
||||
require.Equal(t, "", FirstWord(" "), "Should return an empty string for spaces only")
|
||||
|
||||
// Test with only punctuation
|
||||
require.Equal(t, "", FirstWord(",.!?"), "Should return an empty string for only punctuation")
|
||||
|
||||
// Test with multiple words and punctuation
|
||||
require.Equal(t, "This", FirstWord("This is a sentence!"), "Should return 'This'")
|
||||
|
||||
// Test with non-ASCII characters
|
||||
require.Equal(t, "первое", FirstWord("первое слово"), "Should return the first non-ASCII word 'первое'")
|
||||
|
||||
// Test with hyphenated word
|
||||
require.Equal(t, "Hello", FirstWord("Hello-world!"), "Should return 'Hello' before the hyphen")
|
||||
}
|
||||
|
||||
func TestEscapeHTML(t *testing.T) {
|
||||
// Basic test case
|
||||
require.Equal(t, "<div>", EscapeHTML("<div>"), "Should escape '<' and '>' into '<' and '>'")
|
||||
|
||||
// Test escaping ampersand
|
||||
require.Equal(t, "&hello&", EscapeHTML("&hello&"), "Should escape '&' into '&'")
|
||||
|
||||
// Test escaping mixed characters
|
||||
require.Equal(t, "<div> & text", EscapeHTML("<div> & text"), "Should escape both '<', '>' and '&'")
|
||||
|
||||
// Test string without special characters
|
||||
require.Equal(t, "Hello World", EscapeHTML("Hello World"), "Should return the same string if no HTML special characters")
|
||||
|
||||
// Test string with all special characters
|
||||
require.Equal(t, "&<>", EscapeHTML("&<>"), "Should escape '&', '<', and '>'")
|
||||
|
||||
// Test empty string
|
||||
require.Equal(t, "", EscapeHTML(""), "Should return an empty string when input is empty")
|
||||
|
||||
// Test string with multiple occurrences of the same special characters
|
||||
require.Equal(t, "<<<tag>>>", EscapeHTML("<<<tag>>>"), "Should escape all occurrences of '<' and '>'")
|
||||
}
|
||||
|
||||
func TestStripHTMLTags(t *testing.T) {
|
||||
// Basic test case
|
||||
require.Equal(t, "Hello World", StripHTMLTags("<div>Hello World</div>"), "Should strip basic HTML tags")
|
||||
|
||||
// Test with multiple tags
|
||||
require.Equal(t, "Hello World", StripHTMLTags("<div><p>Hello</p> <b>World</b></div>"), "Should strip multiple HTML tags")
|
||||
|
||||
// Test with self-closing tags
|
||||
require.Equal(t, "Hello", StripHTMLTags("<img src='test.jpg' />Hello"), "Should strip self-closing tags")
|
||||
|
||||
// Test with no HTML tags
|
||||
require.Equal(t, "Plain text", StripHTMLTags("Plain text"), "Should return the same string if no HTML tags are present")
|
||||
|
||||
// Test with empty string
|
||||
require.Equal(t, "", StripHTMLTags(""), "Should return an empty string when input is empty")
|
||||
|
||||
// Test with tag attributes
|
||||
require.Equal(t, "Hello", StripHTMLTags("<a href='https://example.com'>Hello</a>"), "Should strip tags and their attributes")
|
||||
|
||||
// Test with nested tags
|
||||
require.Equal(t, "Nested tags", StripHTMLTags("<div><span>Nested</span> tags</div>"), "Should strip nested HTML tags")
|
||||
|
||||
// Test with special characters
|
||||
require.Equal(t, "1 > 0", StripHTMLTags("1 > 0"), "Should retain special characters like '>' when not in a tag")
|
||||
|
||||
// Test with incomplete tags
|
||||
require.Equal(t, "Text with <tag", StripHTMLTags("Text with <tag"), "Should retain incomplete tags as plain text")
|
||||
}
|
||||
|
||||
func TestReplaceWithPlaceholders(t *testing.T) {
|
||||
// Basic test case
|
||||
str, placeholders := ReplaceWithPlaceholders("Hello World, Hello Universe", "Hello", "placeholder")
|
||||
require.Equal(t, "#placeholder0# World, #placeholder1# Universe", str, "Should replace 'Hello' with placeholders")
|
||||
require.Equal(t, map[string]string{
|
||||
"#placeholder0#": "Hello",
|
||||
"#placeholder1#": "Hello",
|
||||
}, placeholders, "Should map placeholders to their original matches")
|
||||
|
||||
// Test with numbers
|
||||
str, placeholders = ReplaceWithPlaceholders("123-456-7890 and 987-654-3210", `\d{3}-\d{3}-\d{4}`, "phone")
|
||||
require.Equal(t, "#phone0# and #phone1#", str, "Should replace phone numbers with placeholders")
|
||||
require.Equal(t, map[string]string{
|
||||
"#phone0#": "123-456-7890",
|
||||
"#phone1#": "987-654-3210",
|
||||
}, placeholders, "Should map placeholders to phone numbers")
|
||||
|
||||
// Test with special characters
|
||||
str, placeholders = ReplaceWithPlaceholders("email@example.com and test@example.org", `\S+@\S+\.\S+`, "email")
|
||||
require.Equal(t, "#email0# and #email1#", str, "Should replace emails with placeholders")
|
||||
require.Equal(t, map[string]string{
|
||||
"#email0#": "email@example.com",
|
||||
"#email1#": "test@example.org",
|
||||
}, placeholders, "Should map placeholders to emails")
|
||||
|
||||
// Test with no matches
|
||||
str, placeholders = ReplaceWithPlaceholders("No matches here", `\d+`, "num")
|
||||
require.Equal(t, "No matches here", str, "Should return the original string if no matches")
|
||||
require.Empty(t, placeholders, "Should return an empty map if no matches are found")
|
||||
|
||||
// Test with empty string
|
||||
str, placeholders = ReplaceWithPlaceholders("", `\d+`, "num")
|
||||
require.Equal(t, "", str, "Should return an empty string if input is empty")
|
||||
require.Empty(t, placeholders, "Should return an empty map if input is empty")
|
||||
}
|
||||
|
||||
func TestRestoreFromPlaceholders(t *testing.T) {
|
||||
// Basic test case
|
||||
str := "Hello #placeholder0#, Welcome to #placeholder1#!"
|
||||
placeholders := map[string]string{
|
||||
"#placeholder0#": "World",
|
||||
"#placeholder1#": "Earth",
|
||||
}
|
||||
require.Equal(t, "Hello World, Welcome to Earth!", RestoreFromPlaceholders(str, placeholders), "Should restore placeholders to original values")
|
||||
|
||||
// Test with multiple occurrences of the same placeholder
|
||||
str = "#placeholder0# and #placeholder0# are the same"
|
||||
placeholders = map[string]string{
|
||||
"#placeholder0#": "X",
|
||||
}
|
||||
require.Equal(t, "X and X are the same", RestoreFromPlaceholders(str, placeholders), "Should replace all occurrences of the same placeholder")
|
||||
|
||||
// Test with no placeholders
|
||||
str = "No placeholders here"
|
||||
placeholders = map[string]string{}
|
||||
require.Equal(t, "No placeholders here", RestoreFromPlaceholders(str, placeholders), "Should return original string when no placeholders")
|
||||
|
||||
// Test with empty string
|
||||
str = ""
|
||||
placeholders = map[string]string{
|
||||
"#placeholder0#": "X",
|
||||
}
|
||||
require.Equal(t, "", RestoreFromPlaceholders(str, placeholders), "Should return empty string when input is empty")
|
||||
|
||||
// Test with placeholders not in the string
|
||||
str = "No matching placeholders"
|
||||
placeholders = map[string]string{
|
||||
"#placeholder1#": "Y",
|
||||
}
|
||||
require.Equal(t, "No matching placeholders", RestoreFromPlaceholders(str, placeholders), "Should return the original string if placeholder is not found")
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package txt
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf16"
|
||||
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
)
|
||||
|
||||
const ImgPattern = `!\[.*?\]\(.*?\)`
|
||||
|
||||
// TelegramEntitiesToMarkdown converts plain text with Telegram entities (with UTF-16 offsets) to CommonMark Markdown.
|
||||
// Telegram's formatting entities don't take the new lines into account. I.e. if we have a multiline
|
||||
// bold text, it would be referred as a single bold entity, which is not what we want. This function
|
||||
// inserts the necessary closing tags before the new lines and opening tags after the new lines.
|
||||
// https://core.telegram.org/bots/api#messageentity
|
||||
// https://commonmark.org/help/
|
||||
func TelegramEntitiesToMarkdown(text string, messageEntities []tgbotapi.MessageEntity) string {
|
||||
input := []rune(NormNewLines(text))
|
||||
insertions := make(map[int]string)
|
||||
noEscape := make(map[int]*struct{})
|
||||
strct := struct{}{}
|
||||
stopEscape := func(e *tgbotapi.MessageEntity) {
|
||||
for i := e.Offset; i < e.Offset+e.Length; i++ {
|
||||
noEscape[i] = &strct
|
||||
}
|
||||
}
|
||||
|
||||
for _, e := range messageEntities {
|
||||
var before, after string
|
||||
eatNewlines := false // This flag will tell us whether to preserve newlines or not
|
||||
|
||||
if e.IsBold() {
|
||||
before = "**"
|
||||
after = "**"
|
||||
} else if e.IsItalic() {
|
||||
before = "*"
|
||||
after = "*"
|
||||
} else if e.Type == "underline" {
|
||||
before = "__"
|
||||
after = "__"
|
||||
} else if e.Type == "strikethrough" {
|
||||
before = "~"
|
||||
after = "~"
|
||||
} else if e.IsCode() {
|
||||
before = "`"
|
||||
after = "`"
|
||||
stopEscape(&e)
|
||||
} else if e.IsPre() {
|
||||
before = "```" + e.Language + "\n"
|
||||
after = "\n```"
|
||||
eatNewlines = true // For preformatted code, we will eat the newlines
|
||||
stopEscape(&e)
|
||||
} else if e.IsTextLink() {
|
||||
before = "["
|
||||
after = fmt.Sprintf(`](%s)`, e.URL)
|
||||
} else if e.IsURL() {
|
||||
stopEscape(&e)
|
||||
}
|
||||
if before == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
isOpen := false
|
||||
spacesToEat := 0
|
||||
for offset, c := range input[e.Offset : e.Offset+e.Length] {
|
||||
if c == '\n' && !eatNewlines && isOpen {
|
||||
insertions[(e.Offset+offset)-spacesToEat] += after
|
||||
isOpen = false
|
||||
spacesToEat = 0
|
||||
continue
|
||||
}
|
||||
if unicode.IsSpace(c) {
|
||||
spacesToEat++
|
||||
continue
|
||||
}
|
||||
if !isOpen {
|
||||
insertions[e.Offset+offset] += before
|
||||
isOpen = true
|
||||
}
|
||||
spacesToEat = 0
|
||||
}
|
||||
if isOpen {
|
||||
insertions[(e.Offset+e.Length)-spacesToEat] += after
|
||||
}
|
||||
}
|
||||
|
||||
var output []rune
|
||||
utf16pos := 0
|
||||
for _, c := range input {
|
||||
output = append(output, []rune(insertions[utf16pos])...)
|
||||
output = append(output, c)
|
||||
utf16pos += len(utf16.Encode([]rune{c}))
|
||||
}
|
||||
output = append(output, []rune(insertions[utf16pos])...)
|
||||
|
||||
return string(output)
|
||||
}
|
||||
|
||||
func ExtractTextImgsLinks(text string) (txt string, images []string, links map[string]string) {
|
||||
links = make(map[string]string)
|
||||
|
||||
imgRegexp := regexp.MustCompile(`!\[.*?\]\(.*?tg_([^.]+)\..*?\)`)
|
||||
linkRegexp := regexp.MustCompile(`\[.*?\]\((.+?)\)`)
|
||||
wikiLinkRegexp := regexp.MustCompile(`\[\[(.+?)\]\]`)
|
||||
|
||||
// Eat links from lines containing only links
|
||||
text = NormNewLines(text)
|
||||
lines := strings.Split(text, "\n")
|
||||
var processedLines []string
|
||||
for _, line := range lines {
|
||||
trimmedLine := strings.TrimSpace(line)
|
||||
if linkRegexp.MatchString(trimmedLine) && linkRegexp.FindString(trimmedLine) == trimmedLine {
|
||||
matches := linkRegexp.FindStringSubmatch(line)
|
||||
if len(matches) == 2 {
|
||||
content := matches[1]
|
||||
parts := strings.SplitN(content, "|", 2)
|
||||
linkPath := parts[0]
|
||||
linkLabel := strings.TrimSuffix(filepath.Base(linkPath), ".md")
|
||||
links[linkLabel] = linkPath
|
||||
}
|
||||
} else if wikiLinkRegexp.MatchString(trimmedLine) && wikiLinkRegexp.FindString(trimmedLine) == trimmedLine {
|
||||
matches := wikiLinkRegexp.FindStringSubmatch(line)
|
||||
if len(matches) == 2 {
|
||||
content := matches[1]
|
||||
parts := strings.SplitN(content, "|", 2)
|
||||
linkPath := parts[0] + ".md"
|
||||
linkLabel := strings.TrimSuffix(filepath.Base(linkPath), ".md")
|
||||
links[linkLabel] = linkPath
|
||||
}
|
||||
} else {
|
||||
processedLines = append(processedLines, line)
|
||||
}
|
||||
}
|
||||
text = strings.Join(processedLines, "\n")
|
||||
|
||||
// Process images
|
||||
text = imgRegexp.ReplaceAllStringFunc(text, func(match string) string {
|
||||
matches := imgRegexp.FindStringSubmatch(match)
|
||||
if len(matches) == 2 {
|
||||
images = append(images, matches[1])
|
||||
return "🖼"
|
||||
}
|
||||
return match
|
||||
})
|
||||
|
||||
// Process inline links
|
||||
text = linkRegexp.ReplaceAllStringFunc(text, func(match string) string {
|
||||
matches := linkRegexp.FindStringSubmatch(match)
|
||||
if len(matches) == 2 {
|
||||
content := matches[1]
|
||||
parts := strings.SplitN(content, "|", 2)
|
||||
linkPath := parts[0]
|
||||
linkLabel := strings.TrimSuffix(filepath.Base(linkPath), ".md")
|
||||
links[linkLabel] = linkPath
|
||||
|
||||
return "`" + linkLabel + "`"
|
||||
}
|
||||
return match
|
||||
})
|
||||
text = wikiLinkRegexp.ReplaceAllStringFunc(text, func(match string) string {
|
||||
matches := wikiLinkRegexp.FindStringSubmatch(match)
|
||||
if len(matches) == 2 {
|
||||
content := matches[1]
|
||||
parts := strings.SplitN(content, "|", 2)
|
||||
linkPath := parts[0] + ".md"
|
||||
linkLabel := strings.TrimSuffix(filepath.Base(linkPath), ".md")
|
||||
links[linkLabel] = linkPath
|
||||
|
||||
return "`" + linkLabel + "`"
|
||||
}
|
||||
return match
|
||||
})
|
||||
|
||||
return strings.TrimSpace(text), images, links
|
||||
}
|
||||
|
||||
func HasImage(msg string) bool {
|
||||
imgRegexp := regexp.MustCompile(ImgPattern)
|
||||
|
||||
return imgRegexp.MatchString(msg)
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package txt
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBold(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "bold"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "bold", Offset: 0, Length: 4},
|
||||
}
|
||||
|
||||
md := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
r.Equal("**bold**", md)
|
||||
}
|
||||
|
||||
func TestItalic(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "italic"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "italic", Offset: 0, Length: 6},
|
||||
}
|
||||
|
||||
md := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
r.Equal("*italic*", md)
|
||||
}
|
||||
|
||||
func TestBoldAndItalic(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "BoldAndItalic"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "bold", Offset: 0, Length: 13},
|
||||
{Type: "italic", Offset: 0, Length: 13},
|
||||
}
|
||||
|
||||
md := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
r.Equal("***BoldAndItalic***", md)
|
||||
}
|
||||
|
||||
func TestBoldThenItalic(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "bolditalic"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "bold", Offset: 0, Length: 4},
|
||||
{Type: "italic", Offset: 4, Length: 6},
|
||||
}
|
||||
|
||||
md := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
r.Equal("**bold***italic*", md)
|
||||
}
|
||||
|
||||
func TestLink(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "l"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "text_link", Offset: 0, Length: 1, URL: "google.com"},
|
||||
}
|
||||
|
||||
md := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
r.Equal("[l](google.com)", md)
|
||||
}
|
||||
|
||||
func TestMultilineTextWithMarkdown(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "header\nitalic\n\nAlso italic\n\nheader2\nitalic\ncode"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "bold", Offset: 0, Length: 7},
|
||||
{Type: "italic", Offset: 7, Length: 21},
|
||||
{Type: "bold", Offset: 28, Length: 8},
|
||||
{Type: "italic", Offset: 36, Length: 7},
|
||||
{Type: "code", Offset: 43, Length: 4},
|
||||
}
|
||||
|
||||
markdown := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
expectedMarkdown := "**header**\n*italic*\n\n*Also italic*\n\n**header2**\n*italic*\n`code`"
|
||||
r.Equal(expectedMarkdown, markdown)
|
||||
}
|
||||
|
||||
func TestSpacedItalic(t *testing.T) {
|
||||
r := require.New(t)
|
||||
text := "Header\nLeverage one Minute Praising instead"
|
||||
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "italic", Offset: 16, Length: 20},
|
||||
}
|
||||
|
||||
markdown := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
expectedMarkdown := "Header\nLeverage *one Minute Praising* instead"
|
||||
r.Equal(expectedMarkdown, markdown)
|
||||
}
|
||||
|
||||
func TestEmojiInMessageEntities(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "👍b"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "bold", Offset: 2, Length: 1}, // Emoji is 4 bytes or 2 runes
|
||||
}
|
||||
|
||||
md := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
r.Equal("👍**b**", md)
|
||||
}
|
||||
|
||||
func TestSkinEmoji(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "🤘🏾b"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "bold", Offset: 4, Length: 1}, // Tone emoji is 8 bytes or 4 runes
|
||||
}
|
||||
|
||||
md := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
r.Equal("🤘🏾**b**", md)
|
||||
}
|
||||
|
||||
func TestPre(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "line1\nline2"
|
||||
messageEntities := []tgbotapi.MessageEntity{
|
||||
{Type: "pre", Offset: 0, Length: 11},
|
||||
}
|
||||
|
||||
md := TelegramEntitiesToMarkdown(text, messageEntities)
|
||||
r.Equal("```\nline1\nline2\n```", md)
|
||||
}
|
||||
|
||||
func TestDoesntEscapeMD(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "Ask @_a_ __b__ *a* **b** `c` ```multiline```"
|
||||
md := TelegramEntitiesToMarkdown(text, nil)
|
||||
r.Equal("Ask @_a_ __b__ *a* **b** `c` ```multiline```", md)
|
||||
}
|
||||
|
||||
func TestDoesntEscapeBrokenMD(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
text := "Ask @nick_name * `"
|
||||
md := TelegramEntitiesToMarkdown(text, nil)
|
||||
r.Equal("Ask @nick_name * `", md)
|
||||
|
||||
text = "___ *** __ ```"
|
||||
md = TelegramEntitiesToMarkdown(text, nil)
|
||||
r.Equal("___ *** __ ```", md)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_NoImagesOrLinks(t *testing.T) {
|
||||
text := "This is a simple text without images or links."
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "This is a simple text without images or links.", resultText)
|
||||
require.Empty(t, images)
|
||||
require.Empty(t, links)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_WithSingleImage(t *testing.T) {
|
||||
text := "This text includes an image: ."
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "This text includes an image: 🖼.", resultText)
|
||||
require.Equal(t, []string{"BQACAgIAAxkBAAIs"}, images)
|
||||
require.Empty(t, links)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_WithMultipleImages(t *testing.T) {
|
||||
text := "Here are two images:  and ."
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "Here are two images: 🖼 and 🖼.", resultText)
|
||||
require.ElementsMatch(t, []string{"image1", "image2"}, images)
|
||||
require.Empty(t, links)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_WithSingleLink(t *testing.T) {
|
||||
text := "Check this link: [Document](/path/to/document.md)."
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "Check this link: `document`.", resultText)
|
||||
require.Empty(t, images)
|
||||
require.Equal(t, map[string]string{"document": "/path/to/document.md"}, links)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_WithImageAndLink(t *testing.T) {
|
||||
text := "Here is an image:  and a link: [Document](/path/to/doc.md)."
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "Here is an image: 🖼 and a link: `doc`.", resultText)
|
||||
require.Equal(t, []string{"image"}, images)
|
||||
require.Equal(t, map[string]string{"doc": "/path/to/doc.md"}, links)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_WithMultipleLinks(t *testing.T) {
|
||||
text := "Multiple links: [Doc1](/path/to/doc1.md), [Doc2](/path/to/doc2.md)."
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "Multiple links: `doc1`, `doc2`.", resultText)
|
||||
require.Empty(t, images)
|
||||
require.Equal(t, map[string]string{
|
||||
"doc1": "/path/to/doc1.md",
|
||||
"doc2": "/path/to/doc2.md",
|
||||
}, links)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_WithMultipleWikiLinks(t *testing.T) {
|
||||
text := "Multiple links: [[/path/to/doc1|Doc1]], [[/path/to/doc2|Doc2]]."
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "Multiple links: `doc1`, `doc2`.", resultText)
|
||||
require.Empty(t, images)
|
||||
require.Equal(t, map[string]string{
|
||||
"doc1": "/path/to/doc1.md",
|
||||
"doc2": "/path/to/doc2.md",
|
||||
}, links)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_WithBottomLink(t *testing.T) {
|
||||
text := `Text with a bottom link.
|
||||
[Document](/path/to/doc.md)`
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "Text with a bottom link.", resultText)
|
||||
require.Empty(t, images)
|
||||
require.Equal(t, map[string]string{"doc": "/path/to/doc.md"}, links)
|
||||
}
|
||||
|
||||
func TestExtractTextImgsLinks_WithNestedLinksAndImages(t *testing.T) {
|
||||
text := "Complex example with image and links:\n\n[Doc1](/path/to/doc1.md)\n[Doc2](/path/to/doc2.md)"
|
||||
|
||||
resultText, images, links := ExtractTextImgsLinks(text)
|
||||
|
||||
require.Equal(t, "Complex example with image and links:\n🖼", resultText)
|
||||
require.Equal(t, []string{"image"}, images)
|
||||
require.Equal(t, map[string]string{
|
||||
"doc1": "/path/to/doc1.md",
|
||||
"doc2": "/path/to/doc2.md",
|
||||
}, links)
|
||||
}
|
||||
|
||||
func TestHasImage(t *testing.T) {
|
||||
text := "Text with an image: "
|
||||
require.True(t, HasImage(text))
|
||||
|
||||
text = "Text without an image."
|
||||
require.False(t, HasImage(text))
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package plugins
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/exp/slog"
|
||||
)
|
||||
|
||||
const (
|
||||
timeFormat = "02.01.2006 15:04:05"
|
||||
dateFormat = "02.01.2006"
|
||||
)
|
||||
|
||||
var (
|
||||
locationNames = []string{"UTC", "MSK", "CY", "ME"}
|
||||
locations = map[string]*time.Location{
|
||||
"UTC": loadLocation("UTC"),
|
||||
"CY": loadLocation("Asia/Nicosia"),
|
||||
"ME": loadLocation("Europe/Podgorica"),
|
||||
"BG": loadLocation("Europe/Belgrade"),
|
||||
"MSK": loadLocation("Europe/Moscow"),
|
||||
}
|
||||
locationIcons = map[string]string{
|
||||
"UTC": "🕰",
|
||||
"CY": "🏝",
|
||||
"ME": "⛰",
|
||||
"BG": "☕️",
|
||||
"MSK": "🔺",
|
||||
}
|
||||
)
|
||||
|
||||
type WorldClockPlugin struct{}
|
||||
|
||||
func NewWorldClockPlugin() *WorldClockPlugin {
|
||||
return &WorldClockPlugin{}
|
||||
}
|
||||
|
||||
func (p *WorldClockPlugin) CanHandle(msgText string) bool {
|
||||
_, err := p.parseDate(msgText)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
_, err = p.parseTime(msgText)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
_, err = p.parseTimestamp(msgText)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Handle checks if the message is a date, time or timestamp and sends the current time in different timezones
|
||||
func (p *WorldClockPlugin) Handle(msgText string) (string, error) {
|
||||
var message string
|
||||
var err error
|
||||
|
||||
// Try to parse date
|
||||
t, err := p.parseDate(msgText)
|
||||
if err == nil {
|
||||
message = p.buildMessage(t, p.fmtTimestamp)
|
||||
return message, nil
|
||||
}
|
||||
|
||||
// Try to parse time
|
||||
t, err = p.parseTime(msgText)
|
||||
if err == nil {
|
||||
message = p.buildMessage(t, p.fmtTimestamp)
|
||||
return message, nil
|
||||
}
|
||||
|
||||
// Try to parse timestamp
|
||||
t, err = p.parseTimestamp(msgText)
|
||||
if err == nil {
|
||||
message = p.buildMessage(t, p.fmtTime)
|
||||
return message, nil
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (p *WorldClockPlugin) parseTimestamp(message string) (time.Time, error) {
|
||||
timestamp, err := strconv.ParseInt(message, 10, 64)
|
||||
if err == nil && timestamp > 999999 {
|
||||
// Check if it's microseconds (16 digits), milliseconds (13 digits) or seconds (10 digits)
|
||||
if timestamp > 9999999999999 {
|
||||
return time.Unix(timestamp/1000000, (timestamp%1000000)*1000).UTC(), nil
|
||||
} else if timestamp > 9999999999 {
|
||||
return time.Unix(timestamp/1000, (timestamp%1000)*1000000).UTC(), nil
|
||||
}
|
||||
return time.Unix(timestamp, 0).UTC(), nil
|
||||
}
|
||||
return time.Time{}, errors.New("invalid timestamp")
|
||||
}
|
||||
|
||||
func (p *WorldClockPlugin) parseTime(message string) (time.Time, error) {
|
||||
parsedTime, err := time.Parse(timeFormat, message)
|
||||
if err == nil {
|
||||
return parsedTime.UTC(), nil
|
||||
}
|
||||
return time.Time{}, errors.New("invalid time")
|
||||
}
|
||||
|
||||
func (p *WorldClockPlugin) parseDate(message string) (time.Time, error) {
|
||||
parsedDate, err := time.Parse(dateFormat, message)
|
||||
if err == nil {
|
||||
return parsedDate.UTC(), nil
|
||||
}
|
||||
return time.Time{}, errors.New("invalid date")
|
||||
}
|
||||
|
||||
func (p *WorldClockPlugin) buildMessage(t time.Time, formatter func(time.Time) string) string {
|
||||
messageParts := make([]string, len(locations))
|
||||
|
||||
for i, locName := range locationNames {
|
||||
timeInLocation := t.In(locations[locName])
|
||||
formattedTime := formatter(timeInLocation)
|
||||
messageParts[i] = fmt.Sprintf("%v %v %v", locationIcons[locName], formattedTime, locName)
|
||||
}
|
||||
return strings.Join(messageParts, "\n")
|
||||
}
|
||||
|
||||
func (p *WorldClockPlugin) fmtTime(t time.Time) string {
|
||||
return t.Format(timeFormat)
|
||||
}
|
||||
|
||||
func (p *WorldClockPlugin) fmtTimestamp(t time.Time) string {
|
||||
_, offset := t.Zone()
|
||||
timestampInLoc := t.Add(time.Duration(offset) * time.Second).Unix()
|
||||
return strconv.FormatInt(timestampInLoc, 10)
|
||||
}
|
||||
|
||||
func loadLocation(name string) *time.Location {
|
||||
location, err := time.LoadLocation(name)
|
||||
if err != nil {
|
||||
slog.Warn("Error loading location", err)
|
||||
return nil
|
||||
}
|
||||
return location
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package plugins
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestWorldClock_ExecutePlugin_With_Time(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
output, err := worldClockPlugin.Handle("15.06.2023 15:30:00")
|
||||
r.NoError(err)
|
||||
r.Equal("🕰 1686843000 UTC\n🔺 1686853800 MSK\n🏝 1686853800 CY\n⛰ 1686850200 ME\n", output)
|
||||
}
|
||||
|
||||
func TestWorldClock_ExecutePlugin_With_Timestamp(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
output, err := worldClockPlugin.Handle("1686850214")
|
||||
r.NoError(err)
|
||||
r.Equal("🕰 15.06.2023 17:30:14 UTC\n🔺 15.06.2023 20:30:14 MSK\n🏝 15.06.2023 20:30:14 CY\n⛰ 15.06.2023 19:30:14 ME\n", output)
|
||||
}
|
||||
|
||||
func TestWorldClock_ExecutePlugin_With_BotCommand(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
output, err := worldClockPlugin.Handle("cmdShowStart")
|
||||
r.NoError(err)
|
||||
r.Equal("", output)
|
||||
}
|
||||
|
||||
func TestWorldClock_parseTimestamp(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
result, err := worldClockPlugin.parseTimestamp("1686850214")
|
||||
expectedResult := time.Unix(1686850214, 0).UTC()
|
||||
r.Nil(err)
|
||||
r.Equal(expectedResult, result)
|
||||
}
|
||||
|
||||
func TestWorldClock_parseTimestamp_When_InvalidTimestamp(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
_, err := worldClockPlugin.parseTimestamp("ff6480214")
|
||||
r.EqualError(err, "invalid timestamp")
|
||||
}
|
||||
|
||||
func TestWorldClock_parseTime(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
result, err := worldClockPlugin.parseTime("15.06.2023 15:30:00")
|
||||
expectedResult := time.Date(2023, time.June, 15, 15, 30, 0, 0, time.UTC)
|
||||
r.Nil(err)
|
||||
r.Equal(expectedResult, result)
|
||||
}
|
||||
|
||||
func TestWorldClock_parseTime_When_InvalidTime(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
_, err := worldClockPlugin.parseTime("15_06_2023 15:30:00")
|
||||
r.EqualError(err, "invalid time")
|
||||
}
|
||||
|
||||
func TestWorldClock_parseDate(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
result, err := worldClockPlugin.parseDate("15.06.2023")
|
||||
expectedResult := time.Date(2023, time.June, 15, 0, 0, 0, 0, time.UTC)
|
||||
r.Nil(err)
|
||||
r.Equal(expectedResult, result)
|
||||
}
|
||||
|
||||
func TestWorldClock_parseDate_When_InvalidDate(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
_, err := worldClockPlugin.parseDate("41.06.2023")
|
||||
r.EqualError(err, "invalid date")
|
||||
}
|
||||
|
||||
func TestWorldClock_buildMessage(t *testing.T) {
|
||||
r := require.New(t)
|
||||
worldClockPlugin := NewWorldClockPlugin()
|
||||
|
||||
sentTime := time.Date(2023, time.June, 15, 15, 30, 0, 0, time.UTC)
|
||||
result := worldClockPlugin.buildMessage(sentTime, worldClockPlugin.fmtTime)
|
||||
expectedResult := "🕰 15.06.2023 15:30:00 UTC\n🔺 15.06.2023 18:30:00 MSK\n🏝 15.06.2023 18:30:00 CY\n⛰ 15.06.2023 17:30:00 ME\n"
|
||||
r.Equal(expectedResult, result)
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Package stats generates fancy reports
|
||||
// containing completed tasks and habits, checked items and so on
|
||||
package stats
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/zakirullin/files.md/server/db"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
var now = time.Now
|
||||
|
||||
func beginningOfTheDay(t time.Time) time.Time {
|
||||
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
// TODO db is necessary?
|
||||
func TodayReport(userFS *fs.FS, db any, userID int64) (string, error) {
|
||||
files, err := DoneToday(userFS, db, userID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stats.TodayReport: %w", err)
|
||||
}
|
||||
|
||||
var stats []string
|
||||
for _, file := range files {
|
||||
stats = append(stats, fmt.Sprintf("%s <b>%s</b>", emoji(file), fs.DisplayName(file)))
|
||||
}
|
||||
|
||||
archivedFiles, err := userFS.FilesAndDirs(fs.DirArchive)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stats.TodayReport: can't get trashed files: %w", err)
|
||||
}
|
||||
doneTotal := len(archivedFiles)
|
||||
stats = append(stats, fmt.Sprintf("\n📊 %d tasks done in total", doneTotal))
|
||||
|
||||
return strings.Join(stats, "\n"), nil
|
||||
}
|
||||
|
||||
func emoji(filename string) string {
|
||||
if fs.IsChecklistItem(filename) {
|
||||
return "☑️"
|
||||
}
|
||||
|
||||
return "✅"
|
||||
}
|
||||
|
||||
func DoneToday(userFS *fs.FS, db any, userID int64) ([]string, error) {
|
||||
return doneToday(userFS, db, userID, false)
|
||||
}
|
||||
|
||||
func DoneTodayScheduled(userFS *fs.FS, db *db.DB, userID int64) ([]string, error) {
|
||||
return doneToday(userFS, db, userID, true)
|
||||
}
|
||||
|
||||
func doneToday(userFS *fs.FS, db any, userID int64, withScheduled bool) ([]string, error) {
|
||||
files, err := userFS.FilesAndDirs(fs.DirArchive)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stats.DoneTasks: %w", err)
|
||||
}
|
||||
|
||||
var todayFiles []fs.File
|
||||
for _, task := range files {
|
||||
if task.Ctime > beginningOfTheDay(now()).Unix() {
|
||||
todayFiles = append(todayFiles, task)
|
||||
}
|
||||
}
|
||||
|
||||
//sch, err := db.Schedule(userID)
|
||||
//if err != nil {
|
||||
// return nil, fmt.Errorf("stats.DoneTasks: %w", err)
|
||||
//}
|
||||
|
||||
var done []string
|
||||
for _, todayFile := range todayFiles {
|
||||
done = append(done, todayFile.DisplayName)
|
||||
}
|
||||
|
||||
return done, nil
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zakirullin/files.md/server/db"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
func TestDoneToday(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
saved := fs.Ctime
|
||||
defer func() {
|
||||
fs.Ctime = saved
|
||||
}()
|
||||
fs.Ctime = func(fi os.FileInfo) int64 {
|
||||
return 1
|
||||
}
|
||||
|
||||
savedNow := now
|
||||
defer func() {
|
||||
now = savedNow
|
||||
}()
|
||||
now = func() time.Time {
|
||||
return time.Unix(0, 0)
|
||||
}
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
err = userFS.Write("archive", "a.md", "")
|
||||
r.NoError(err)
|
||||
|
||||
database := db.NewDB(-1)
|
||||
r.NoError(err)
|
||||
|
||||
tasks, err := DoneToday(userFS, database, -1)
|
||||
r.NoError(err)
|
||||
|
||||
r.Equal([]string{"A"}, tasks)
|
||||
}
|
||||
|
||||
//func TestDoneTodayExcludeScheduled(t *testing.T) {
|
||||
// r := require.New(t)
|
||||
//
|
||||
// saved := fs.Ctime
|
||||
// defer func() {
|
||||
// fs.Ctime = saved
|
||||
// }()
|
||||
// fs.Ctime = func(fi os.FileInfo) int64 {
|
||||
// return 1
|
||||
// }
|
||||
//
|
||||
// savedNow := now
|
||||
// defer func() {
|
||||
// now = savedNow
|
||||
// }()
|
||||
// now = func() time.Time {
|
||||
// return time.Unix(0, 0)
|
||||
// }
|
||||
//
|
||||
// userFS, _ := fs.NewFS("/", afero.NewMemMapFs())
|
||||
// err := userFS.Write("archive", "a.md", "")
|
||||
// r.NoError(err)
|
||||
//
|
||||
// userDB := db.NewDB()
|
||||
// err = db.AddToSchedule(-1, "a.md", 1, "cron")
|
||||
// r.NoError(err)
|
||||
//
|
||||
// tasks, err := DoneToday(fs, db, -1)
|
||||
// r.NoError(err)
|
||||
//
|
||||
// r.Empty(tasks)
|
||||
//}
|
||||
|
||||
//func TestDoneTodayScheduled(t *testing.T) {
|
||||
// r := require.New(t)
|
||||
//
|
||||
// saved := fs.Ctime
|
||||
// defer func() {
|
||||
// fs.Ctime = saved
|
||||
// }()
|
||||
// fs.Ctime = func(fi os.FileInfo) int64 {
|
||||
// return 1
|
||||
// }
|
||||
//
|
||||
// savedNow := now
|
||||
// defer func() {
|
||||
// now = savedNow
|
||||
// }()
|
||||
// now = func() time.Time {
|
||||
// return time.Unix(0, 0)
|
||||
// }
|
||||
//
|
||||
// fs, _ := fs.NewFS("/", afero.NewMemMapFs())
|
||||
// err := fs.Put("archive", "a.md", "")
|
||||
// r.NoError(err)
|
||||
// err = fs.Put("archive", "b.md", "")
|
||||
// r.NoError(err)
|
||||
//
|
||||
// redis, err := miniredis.Run()
|
||||
// if err != nil {
|
||||
// panic(fmt.Sprintf("Can't create Redis: %s\n", err))
|
||||
// }
|
||||
// defer func() {
|
||||
// redis.Close()
|
||||
// }()
|
||||
//
|
||||
// db := dbpkg.NewDB(redis)
|
||||
// err = db.AddToSchedule(-1, "a.md", 1, "cron")
|
||||
// r.NoError(err)
|
||||
//
|
||||
// tasks, err := DoneTodayScheduled(fs, db, -1)
|
||||
// r.NoError(err)
|
||||
// r.Equal([]string{"A"}, tasks)
|
||||
//}
|
||||
@@ -0,0 +1,39 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/acme/autocert"
|
||||
)
|
||||
|
||||
func certServer(logger *log.Logger, certDir string, hosts ...string) *autocert.Manager {
|
||||
autocertManager := autocert.Manager{
|
||||
Prompt: autocert.AcceptTOS,
|
||||
HostPolicy: autocert.HostWhitelist(hosts...),
|
||||
Cache: autocert.DirCache(certDir),
|
||||
}
|
||||
|
||||
// Listen for HTTP requests on port 80 in a new goroutine. Use
|
||||
// autocertManager.HTTPHandler(nil) as the handler. This will send ACME
|
||||
// "http-01" challenge responses as necessary, and 302 redirect all other
|
||||
// requests to HTTPS.
|
||||
go func() {
|
||||
srv := &http.Server{
|
||||
Addr: ":80",
|
||||
Handler: autocertManager.HTTPHandler(nil),
|
||||
IdleTimeout: time.Minute,
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
ErrorLog: logger,
|
||||
}
|
||||
|
||||
err := srv.ListenAndServe()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
return &autocertManager
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
)
|
||||
|
||||
const (
|
||||
Rename = "ren"
|
||||
Delete = "del"
|
||||
)
|
||||
|
||||
var lock sync.RWMutex
|
||||
|
||||
func LogRename(time int64, oldPath, newPath string) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
file, err := os.OpenFile(path.Join(config.ServerCfg.WorkingDir, "fslog"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
oldPath = url.QueryEscape(oldPath)
|
||||
newPath = url.QueryEscape(newPath)
|
||||
record := fmt.Sprintf("%d %s %s %s\n", time, Rename, oldPath, newPath)
|
||||
|
||||
file.WriteString(record)
|
||||
file.Sync()
|
||||
}
|
||||
|
||||
func LogDelete(time int64, filepath string) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
file, err := os.OpenFile(path.Join(config.ServerCfg.WorkingDir, "fslog"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
filepath = url.QueryEscape(filepath)
|
||||
record := fmt.Sprintf("%d %s %s\n", time, Delete, filepath)
|
||||
|
||||
file.WriteString(record)
|
||||
file.Sync()
|
||||
}
|
||||
|
||||
// RenamesLog reads the file system renames log and returns a map of:
|
||||
// newPath -> oldPath
|
||||
// AfterTimestamp is inclusive.
|
||||
func RenamesLog(userID, afterTimestamp int64) map[string]string {
|
||||
lock.RLock()
|
||||
defer lock.RUnlock()
|
||||
|
||||
// TODO can we tolerate errors? The worst that happens are duplicates on client side
|
||||
file, err := os.Open(path.Join(config.ServerCfg.WorkingDir, "fslog"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
logEntries := make(map[string]string)
|
||||
scanner := bufio.NewScanner(file)
|
||||
userPathPrefix := path.Join(config.ServerCfg.StorageDir, fmt.Sprintf("%d", userID)) + "/"
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
var timestamp int64
|
||||
var op, oldPath, newPath string
|
||||
n, err := fmt.Sscanf(line, "%d %s %s %s", ×tamp, &op, &oldPath, &newPath)
|
||||
if op != Rename {
|
||||
continue
|
||||
}
|
||||
if err != nil || n != 4 || timestamp < afterTimestamp {
|
||||
continue
|
||||
}
|
||||
oldPath, err = url.QueryUnescape(oldPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
newPath, err = url.QueryUnescape(newPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// TODO exclude ../ from log to prevent Filename Traversal attack
|
||||
// Or do we need it? Log rename only logs bot's renames, which are bounded by user folders.
|
||||
// And if an attacker get access to the log, he would be able to read files anyway.
|
||||
|
||||
if !strings.HasPrefix(oldPath, userPathPrefix) || !strings.HasPrefix(newPath, userPathPrefix) {
|
||||
continue
|
||||
}
|
||||
oldPath = strings.TrimPrefix(oldPath, userPathPrefix)
|
||||
newPath = strings.TrimPrefix(newPath, userPathPrefix)
|
||||
|
||||
logEntries[newPath] = oldPath
|
||||
}
|
||||
|
||||
return logEntries
|
||||
}
|
||||
|
||||
// DeletesLog reads the file system deletes log and returns a map of:
|
||||
// path -> deletedAt unix timestamp
|
||||
// AfterTimestamp is inclusive. If a path was deleted multiple times,
|
||||
// the latest timestamp wins.
|
||||
func DeletesLog(userID, afterTimestamp int64) map[string]int64 {
|
||||
lock.RLock()
|
||||
defer lock.RUnlock()
|
||||
|
||||
file, err := os.Open(path.Join(config.ServerCfg.WorkingDir, "fslog"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
logEntries := make(map[string]int64)
|
||||
scanner := bufio.NewScanner(file)
|
||||
userPathPrefix := path.Join(config.ServerCfg.StorageDir, fmt.Sprintf("%d", userID)) + "/"
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
var timestamp int64
|
||||
var op, filepath string
|
||||
n, err := fmt.Sscanf(line, "%d %s %s", ×tamp, &op, &filepath)
|
||||
if op != Delete {
|
||||
continue
|
||||
}
|
||||
if err != nil || n != 3 || timestamp < afterTimestamp {
|
||||
continue
|
||||
}
|
||||
filepath, err = url.QueryUnescape(filepath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(filepath, userPathPrefix) {
|
||||
continue
|
||||
}
|
||||
filepath = strings.TrimPrefix(filepath, userPathPrefix)
|
||||
|
||||
if existing, ok := logEntries[filepath]; !ok || timestamp > existing {
|
||||
logEntries[filepath] = timestamp
|
||||
}
|
||||
}
|
||||
|
||||
return logEntries
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/rivo/uniseg"
|
||||
)
|
||||
|
||||
const (
|
||||
// Journal day headers used to be `#### 23 May, Friday`; new entries are
|
||||
// `## 23 May, Friday`. Match both so cross-format journals still merge.
|
||||
header = `^(####|##) \d+ \w+, \w+`
|
||||
)
|
||||
|
||||
// Merge combines two strings (s1 and s2) by identifying longest sequences of common lines.
|
||||
//
|
||||
// The algorithm:
|
||||
// 1) Splits both inputs into lines
|
||||
// 2) Uses dynamic programming to find the longest common subsequence (LCS) between every two lines
|
||||
// 3) Constructs a merged result that preserves all unique content from both strings
|
||||
// 4) Maintains the original order of content from both strings
|
||||
// TODO add support for json merging
|
||||
func Merge(s1, s2 string) string {
|
||||
if len(s1) == 0 {
|
||||
return s2
|
||||
}
|
||||
if len(s2) == 0 {
|
||||
return s1
|
||||
}
|
||||
lines1 := strings.Split(s1, "\n")
|
||||
lines2 := strings.Split(s2, "\n")
|
||||
|
||||
// Dynamical programming table containing the longest common prefix for each pair.
|
||||
lcsLength := make([][]int, len(lines1)+1)
|
||||
for i := range lcsLength {
|
||||
lcsLength[i] = make([]int, len(lines2)+1)
|
||||
}
|
||||
|
||||
// Fill the lcsLength table.
|
||||
for i := 1; i <= len(lines1); i++ {
|
||||
for j := 1; j <= len(lines2); j++ {
|
||||
if lines1[i-1] == lines2[j-1] {
|
||||
lcsLength[i][j] = lcsLength[i-1][j-1] + 1
|
||||
} else {
|
||||
lcsLength[i][j] = max(lcsLength[i-1][j], lcsLength[i][j-1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build the merged result.
|
||||
result := backtrack(lines1, lines2, lcsLength, len(lines1), len(lines2))
|
||||
result = mergeEmojisInJournalHeaders(result)
|
||||
|
||||
return strings.Join(result, "\n")
|
||||
}
|
||||
|
||||
// backtrack performs backtracking through the dynamic programming table lcsLength
|
||||
// to construct the merged result based on the longest common subsequence (LCS).
|
||||
func backtrack(lines1, lines2 []string, lcsLength [][]int, i, j int) []string {
|
||||
if i == 0 && j == 0 {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
if i == 0 {
|
||||
return append(backtrack(lines1, lines2, lcsLength, i, j-1), lines2[j-1])
|
||||
}
|
||||
|
||||
if j == 0 {
|
||||
return append(backtrack(lines1, lines2, lcsLength, i-1, j), lines1[i-1])
|
||||
}
|
||||
|
||||
// If the current lines are the same, include it only once.
|
||||
if lines1[i-1] == lines2[j-1] {
|
||||
return append(backtrack(lines1, lines2, lcsLength, i-1, j-1), lines1[i-1])
|
||||
}
|
||||
|
||||
// Choose the direction with the longer common subsequence.
|
||||
if lcsLength[i-1][j] > lcsLength[i][j-1] {
|
||||
return append(backtrack(lines1, lines2, lcsLength, i-1, j), lines1[i-1])
|
||||
} else {
|
||||
return append(backtrack(lines1, lines2, lcsLength, i, j-1), lines2[j-1])
|
||||
}
|
||||
}
|
||||
|
||||
// Headers like this should be merged:
|
||||
// ## 23 May, Friday
|
||||
// ## 23 May, Friday 🤸
|
||||
// ## 23 May, Friday 🤸🍽
|
||||
// ## 23 May, Friday 🤸🍽💪
|
||||
// ## 23 May, Friday 🤸🍽💪💧
|
||||
// ## 23 May, Friday 🤸🍽💪💧🚶♂️
|
||||
func mergeEmojisInJournalHeaders(lines []string) []string {
|
||||
var mergedLines []string
|
||||
groups := groupConsecutiveHeaders(lines)
|
||||
for _, group := range groups {
|
||||
if len(group) == 1 {
|
||||
mergedLines = append(mergedLines, group[0])
|
||||
continue
|
||||
}
|
||||
|
||||
possibleEmojis := regexp.MustCompile(` [^\w\s\p{P}]+$`)
|
||||
date := possibleEmojis.ReplaceAllString(group[0], "")
|
||||
prefixIsSame := true
|
||||
for _, line := range group {
|
||||
emojis := possibleEmojis.FindString(line)
|
||||
if date+emojis != line {
|
||||
prefixIsSame = false
|
||||
}
|
||||
}
|
||||
// If at least one line from group doesn't start with the same date, we can't merge them.
|
||||
if !prefixIsSame {
|
||||
mergedLines = append(mergedLines, group...)
|
||||
continue
|
||||
}
|
||||
|
||||
foundEmojis := ""
|
||||
for _, line := range group {
|
||||
foundEmojis += strings.TrimSpace(possibleEmojis.FindString(line))
|
||||
}
|
||||
if foundEmojis != "" {
|
||||
foundEmojis = " " + unique(foundEmojis)
|
||||
}
|
||||
mergedLines = append(mergedLines, date+foundEmojis)
|
||||
}
|
||||
|
||||
return mergedLines
|
||||
}
|
||||
|
||||
func groupConsecutiveHeaders(lines []string) [][]string {
|
||||
re := regexp.MustCompile(header)
|
||||
var groups [][]string
|
||||
i := 0
|
||||
for i < len(lines) {
|
||||
if re.MatchString(lines[i]) {
|
||||
var group []string
|
||||
for i < len(lines) && re.MatchString(lines[i]) {
|
||||
group = append(group, lines[i])
|
||||
i++
|
||||
}
|
||||
groups = append(groups, group)
|
||||
} else {
|
||||
groups = append(groups, []string{lines[i]})
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
// unique returns a string containing unique unicode graphemes from both input strings.
|
||||
// The order is preserved.
|
||||
func unique(s string) string {
|
||||
var uniq string
|
||||
graphemes := uniseg.NewGraphemes(s)
|
||||
for graphemes.Next() {
|
||||
g := graphemes.Str()
|
||||
if !strings.Contains(uniq, g) {
|
||||
uniq += g
|
||||
}
|
||||
}
|
||||
|
||||
return uniq
|
||||
}
|
||||
@@ -0,0 +1,933 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMergePrefixCases(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "line 1\nline 2"
|
||||
modified := "line 1\nline 2\nline 3\nline 4"
|
||||
r.Equal(modified, Merge(original, modified))
|
||||
r.Equal(modified, Merge(modified, original))
|
||||
}
|
||||
|
||||
func TestMergeCommonPrefixDifferentSuffixes(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Both have common prefix but different additional lines
|
||||
original := "line 1\nline 2\nline 3\nline original 4"
|
||||
modified := "line 1\nline 2\nline 3\nline modified 4"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("line 1\nline 2\nline 3\nline original 4\nline modified 4", merged)
|
||||
}
|
||||
|
||||
func TestMergeDifferentPrefixCommonSuffix(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Both have different prefixes but common suffix
|
||||
original := "line original 1\nline original 2\nline 3"
|
||||
modified := "new\nline original 1\nline original 2\nline 3"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("new\nline original 1\nline original 2\nline 3", merged, "Should merge lines before common suffix")
|
||||
}
|
||||
|
||||
func TestMergeDivergentBody(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Divergent content with common prefix and suffix
|
||||
original := "header\nheader\noriginal A\noriginal B\nfooter\nfooter"
|
||||
modified := "header\nheader\nmodified X\nmodified Y\nfooter\nfooter"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("header\nheader\noriginal A\noriginal B\nmodified X\nmodified Y\nfooter\nfooter", merged)
|
||||
}
|
||||
|
||||
func TestMergeSameHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
result := Merge("#### 23 May, Saturday", "#### 23 May, Saturday")
|
||||
r.Equal("#### 23 May, Saturday", result)
|
||||
}
|
||||
|
||||
func TestMergeDivergentContent(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Complete divergence with small common prefix
|
||||
original := "header\noriginal A\noriginal B"
|
||||
modified := "header\nmodified X\nmodified Y"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("header\noriginal A\noriginal B\nmodified X\nmodified Y", merged)
|
||||
}
|
||||
|
||||
func TestMergeEmptyStrings(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
r.Equal("", Merge("", ""), "Empty strings should merge to empty string")
|
||||
r.Equal("content", Merge("", "content"), "Empty original should return modified")
|
||||
r.Equal("content", Merge("content", ""), "Empty modified should return original")
|
||||
}
|
||||
|
||||
func TestMergeTrailingNewlines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "line 1\nline 2\n"
|
||||
modified := "line 1\nline 2\nline 3\n"
|
||||
r.Equal(modified, Merge(original, modified), "Should handle trailing newlines correctly")
|
||||
}
|
||||
|
||||
func TestMergeDivergentChars(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "abc"
|
||||
modified := "adc"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("abc\nadc", merged)
|
||||
}
|
||||
|
||||
func TestJournal(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
server := "1 April\nfelt good\nate good\n2 April\nslept not so good"
|
||||
client := "1 April\nfelt good\n2 April\nslept not so good\nwent for hiking"
|
||||
merged := Merge(server, client)
|
||||
r.Equal("1 April\nfelt good\nate good\n2 April\nslept not so good\nwent for hiking", merged)
|
||||
}
|
||||
|
||||
func TestMergeHeaders(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
headers := []string{"#### 23 May, Friday 🤸♂️🍽💪💧", "#### 23 May, Friday 🤸♂️🍽💪", "#### 23 May, Friday 🤸♂️"}
|
||||
merged := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 🤸♂️🍽💪💧"}, merged)
|
||||
}
|
||||
|
||||
func TestMergeHeadersReversed(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
headers := []string{"#### 23 May, Friday 🤸♂️", "#### 23 May, Friday 🤸♂️🍽💪", "#### 23 May, Friday 🤸♂️🍽💪💧"}
|
||||
merged := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 🤸♂️🍽💪💧"}, merged)
|
||||
}
|
||||
|
||||
func TestMergeHeadersWithDifferentEmojis(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
headers := []string{"#### 23 May, Friday 🤸♂️🍽💪💧", "#### 23 May, Friday 🤸♂️🍽💪📵🚶♂️"}
|
||||
merged := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 🤸♂️🍽💪💧📵🚶♂️"}, merged)
|
||||
}
|
||||
|
||||
func TestMergeHeadersNoEmoji(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
headers := []string{"#### 23 May, Friday", "#### 23 May, Friday 💪"}
|
||||
merged := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 💪"}, merged)
|
||||
|
||||
headers = []string{"#### 23 May, Saturday", "#### 23 May, Saturday"}
|
||||
merged = mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Saturday"}, merged)
|
||||
}
|
||||
|
||||
// AI-gen tests
|
||||
|
||||
func TestMergeCompletelyDifferent(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "apple\nbanana\ncherry"
|
||||
modified := "dog\nelephant\nfox"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("apple\nbanana\ncherry\ndog\nelephant\nfox", merged)
|
||||
}
|
||||
|
||||
func TestMergeRepeatedLines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "repeat\nrepeat\nunique1"
|
||||
modified := "repeat\nrepeat\nunique2"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("repeat\nrepeat\nunique1\nunique2", merged)
|
||||
}
|
||||
|
||||
func TestMergeWithBlankLines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "line1\n\nline3"
|
||||
modified := "line1\nline2\n\nline3"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("line1\nline2\n\nline3", merged)
|
||||
}
|
||||
|
||||
func TestMergeMultipleBlankLines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "start\n\n\nend"
|
||||
modified := "start\nmiddle\n\n\nend"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("start\nmiddle\n\n\nend", merged)
|
||||
}
|
||||
|
||||
func TestMergeOnlyBlankLines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "\n\n"
|
||||
modified := "\n\n\n"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("\n\n\n", merged)
|
||||
}
|
||||
|
||||
func TestMergeSingleLineStrings(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
r.Equal("hello", Merge("hello", "hello"))
|
||||
r.Equal("hello\nworld", Merge("hello", "world"))
|
||||
r.Equal("world\nhello", Merge("world", "hello"))
|
||||
}
|
||||
|
||||
func TestMergeVeryLongCommonPrefix(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
commonLines := make([]string, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
commonLines[i] = fmt.Sprintf("common line %d", i)
|
||||
}
|
||||
commonPrefix := strings.Join(commonLines, "\n")
|
||||
|
||||
original := commonPrefix + "\noriginal ending"
|
||||
modified := commonPrefix + "\nmodified ending"
|
||||
merged := Merge(original, modified)
|
||||
expected := commonPrefix + "\noriginal ending\nmodified ending"
|
||||
r.Equal(expected, merged)
|
||||
}
|
||||
|
||||
func TestMergeVeryLongCommonSuffix(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
commonLines := make([]string, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
commonLines[i] = fmt.Sprintf("common line %d", i)
|
||||
}
|
||||
commonSuffix := strings.Join(commonLines, "\n")
|
||||
|
||||
original := "original start\n" + commonSuffix
|
||||
modified := "modified start\n" + commonSuffix
|
||||
merged := Merge(original, modified)
|
||||
expected := "original start\nmodified start\n" + commonSuffix
|
||||
r.Equal(expected, merged)
|
||||
}
|
||||
|
||||
func TestMergeNestedCommonSubsequences(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Complex case with multiple common subsequences
|
||||
original := "A\nB\nC\nX\nD\nE\nY\nF"
|
||||
modified := "A\nB\nZ\nC\nD\nE\nW\nF"
|
||||
merged := Merge(original, modified)
|
||||
// Should preserve the LCS while adding unique content
|
||||
r.Contains(merged, "A")
|
||||
r.Contains(merged, "B")
|
||||
r.Contains(merged, "C")
|
||||
r.Contains(merged, "D")
|
||||
r.Contains(merged, "E")
|
||||
r.Contains(merged, "F")
|
||||
r.Contains(merged, "X")
|
||||
r.Contains(merged, "Y")
|
||||
r.Contains(merged, "Z")
|
||||
r.Contains(merged, "W")
|
||||
}
|
||||
|
||||
func TestMergeWithSpecialCharacters(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "line with\ttabs\nline with spaces"
|
||||
modified := "line with\ttabs\nline with multiple spaces"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("line with\ttabs\nline with spaces\nline with multiple spaces", merged)
|
||||
}
|
||||
|
||||
func TestMergeUnicodeContent(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
original := "Hello 世界\n🌍 Earth"
|
||||
modified := "Hello 世界\n🌍 Earth\n🚀 Space"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal("Hello 世界\n🌍 Earth\n🚀 Space", merged)
|
||||
}
|
||||
|
||||
func TestMergeVeryLongLines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
longLine := strings.Repeat("a", 10000)
|
||||
original := longLine + "\nshort"
|
||||
modified := longLine + "\ndifferent"
|
||||
merged := Merge(original, modified)
|
||||
r.Equal(longLine+"\nshort\ndifferent", merged)
|
||||
}
|
||||
|
||||
func TestMergeIdenticalContent(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
content := "line1\nline2\nline3\nline4\nline5"
|
||||
r.Equal(content, Merge(content, content))
|
||||
}
|
||||
|
||||
func TestMergeOneIsSubsetOfOther(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
subset := "line2\nline4"
|
||||
superset := "line1\nline2\nline3\nline4\nline5"
|
||||
|
||||
// Test both directions
|
||||
r.Equal(superset, Merge(subset, superset))
|
||||
r.Equal(superset, Merge(superset, subset))
|
||||
}
|
||||
|
||||
func TestMergeAlternatingPattern(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Alternating common and unique lines
|
||||
original := "common1\nunique1\ncommon2\nunique2\ncommon3"
|
||||
modified := "common1\ndifferent1\ncommon2\ndifferent2\ncommon3"
|
||||
merged := Merge(original, modified)
|
||||
|
||||
r.Contains(merged, "common1")
|
||||
r.Contains(merged, "common2")
|
||||
r.Contains(merged, "common3")
|
||||
r.Contains(merged, "unique1")
|
||||
r.Contains(merged, "unique2")
|
||||
r.Contains(merged, "different1")
|
||||
r.Contains(merged, "different2")
|
||||
}
|
||||
|
||||
func TestMergeRealWorldScenario(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Simulating a config file merge
|
||||
original := `# Configuration file
|
||||
version: 1.0
|
||||
database:
|
||||
host: localhost
|
||||
port: 5432
|
||||
logging:
|
||||
level: info`
|
||||
|
||||
modified := `# Configuration file
|
||||
version: 1.0
|
||||
database:
|
||||
host: localhost
|
||||
port: 5432
|
||||
timeout: 30
|
||||
logging:
|
||||
level: debug
|
||||
file: app.log`
|
||||
|
||||
merged := Merge(original, modified)
|
||||
|
||||
// Should contain all unique lines from both
|
||||
r.Contains(merged, "# Configuration file")
|
||||
r.Contains(merged, "version: 1.0")
|
||||
r.Contains(merged, "database:")
|
||||
r.Contains(merged, " host: localhost")
|
||||
r.Contains(merged, " port: 5432")
|
||||
r.Contains(merged, " timeout: 30")
|
||||
r.Contains(merged, "logging:")
|
||||
r.Contains(merged, " level: info")
|
||||
r.Contains(merged, " level: debug")
|
||||
r.Contains(merged, " file: app.log")
|
||||
}
|
||||
|
||||
func TestMergeJournalWithTasks(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// More complex journal scenario
|
||||
original := `#### 24 May, Sunday
|
||||
Morning routine
|
||||
- Coffee ☕
|
||||
- Exercise 💪
|
||||
Evening reflection
|
||||
- Good day overall`
|
||||
|
||||
modified := `#### 24 May, Sunday
|
||||
Morning routine
|
||||
- Coffee ☕
|
||||
- Read news 📰
|
||||
- Exercise 💪
|
||||
Afternoon work
|
||||
- Team meeting
|
||||
Evening reflection
|
||||
- Good day overall
|
||||
- Grateful for sunshine`
|
||||
|
||||
merged := Merge(original, modified)
|
||||
|
||||
// Should preserve the structure while adding new content
|
||||
r.Contains(merged, "#### 24 May, Sunday")
|
||||
r.Contains(merged, "Morning routine")
|
||||
r.Contains(merged, "- Coffee ☕")
|
||||
r.Contains(merged, "- Exercise 💪")
|
||||
r.Contains(merged, "- Read news 📰")
|
||||
r.Contains(merged, "Afternoon work")
|
||||
r.Contains(merged, "- Team meeting")
|
||||
r.Contains(merged, "Evening reflection")
|
||||
r.Contains(merged, "- Good day overall")
|
||||
r.Contains(merged, "- Grateful for sunshine")
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_SingleHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Single header with emojis
|
||||
headers := []string{"#### 23 May, Friday 🤸♂️🍽💪"}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 🤸♂️🍽💪"}, result)
|
||||
|
||||
// Single header without emojis
|
||||
headers = []string{"#### 23 May, Friday"}
|
||||
result = mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday"}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_MultipleHeadersSameDate(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Multiple headers with same date, different emojis
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 23 May, Friday 🍽💪",
|
||||
"#### 23 May, Friday 💧",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 🤸♂️🍽💪💧"}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_OverlappingEmojis(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Headers with overlapping emojis - should deduplicate
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 🤸♂️🍽💪",
|
||||
"#### 23 May, Friday 🍽💪💧",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 🤸♂️🍽💪💧"}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_DifferentDates(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Headers with different dates - should not merge
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 24 May, Saturday 🍽💪",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 24 May, Saturday 🍽💪",
|
||||
}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_PartialDateMatch(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Headers where one starts with part of another's date - should not merge
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 23 May, Friday evening 🍽💪",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
// Should not merge because "#### 23 May, Friday evening" doesn't start with "#### 23 May, Friday"
|
||||
r.Equal([]string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 23 May, Friday evening 🍽💪",
|
||||
}, result)
|
||||
}
|
||||
|
||||
func TestMergeTwoNonHeaders(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Headers where one starts with part of another's date - should not merge
|
||||
headers := []string{
|
||||
"#### 🤸♂️",
|
||||
"#### 🍽💪",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{
|
||||
"#### 🤸♂️",
|
||||
"#### 🍽💪",
|
||||
}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_NoEmojis(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Multiple headers with same date but no emojis
|
||||
headers := []string{
|
||||
"#### 23 May, Friday",
|
||||
"#### 23 May, Friday",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday"}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_MixedEmojiAndNoEmoji(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Mix of headers with and without emojis
|
||||
headers := []string{
|
||||
"#### 23 May, Friday",
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 23 May, Friday 🍽💪",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 🤸♂️🍽💪"}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_NonHeaderLines(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Mix of headers and non-headers
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"This is not a header",
|
||||
"Neither is this",
|
||||
"#### 24 May, Saturday 🍽💪",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"This is not a header",
|
||||
"Neither is this",
|
||||
"#### 24 May, Saturday 🍽💪",
|
||||
}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_ConsecutiveGroupsWithNonHeaders(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Multiple groups separated by non-headers
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 23 May, Friday 🍽",
|
||||
"Some content here",
|
||||
"#### 24 May, Saturday 💪",
|
||||
"#### 24 May, Saturday 💧",
|
||||
"More content",
|
||||
"#### 25 May, Sunday 🚶♂️",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{
|
||||
"#### 23 May, Friday 🤸♂️🍽",
|
||||
"Some content here",
|
||||
"#### 24 May, Saturday 💪💧",
|
||||
"More content",
|
||||
"#### 25 May, Sunday 🚶♂️",
|
||||
}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_ComplexEmojis(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test with complex emojis (multi-byte, skin tones, etc.)
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 👨💻",
|
||||
"#### 23 May, Friday 🏃♂️",
|
||||
"#### 23 May, Friday 👍🏽",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 👨💻🏃♂️👍🏽"}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_EmojiOrder(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test that emoji order is preserved
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 🥇",
|
||||
"#### 23 May, Friday 🥈",
|
||||
"#### 23 May, Friday 🥉",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday 🥇🥈🥉"}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_EmptyInput(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
result := mergeEmojisInJournalHeaders([]string{})
|
||||
r.Empty(result)
|
||||
|
||||
result = mergeEmojisInJournalHeaders(nil)
|
||||
r.Empty(result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_BugFix_NonMergeable(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// This test specifically targets the bug you found
|
||||
// When headers can't be merged, foundEmojis is accumulated but not used
|
||||
headers := []string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 24 May, Saturday 🍽💪", // Different date
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
|
||||
// Should return original headers unchanged, not merged
|
||||
r.Equal([]string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 24 May, Saturday 🍽💪",
|
||||
}, result)
|
||||
|
||||
// The bug would have been that foundEmojis was being accumulated
|
||||
// even when the headers couldn't be merged due to different dates
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_SpecialCharacters(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Test with punctuation and special characters
|
||||
headers := []string{
|
||||
"#### 23 May, Friday! 🎉",
|
||||
"#### 23 May, Friday! 🎊",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
r.Equal([]string{"#### 23 May, Friday! 🎉🎊"}, result)
|
||||
}
|
||||
|
||||
func TestMergeEmojisInJournalHeaders_RealWorldScenario(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Realistic journal headers as they might appear
|
||||
headers := []string{
|
||||
"#### 28 May, Wednesday 🌅", // morning entry
|
||||
"#### 28 May, Wednesday 💼", // work entry
|
||||
"#### 28 May, Wednesday 🍽️", // meal entry
|
||||
"#### 28 May, Wednesday 🌙", // evening entry
|
||||
"Some journal content here",
|
||||
"#### 29 May, Thursday ☀️",
|
||||
"#### 29 May, Thursday 🏃♂️💪",
|
||||
}
|
||||
result := mergeEmojisInJournalHeaders(headers)
|
||||
expected := []string{
|
||||
"#### 28 May, Wednesday 🌅💼🍽️🌙",
|
||||
"Some journal content here",
|
||||
"#### 29 May, Thursday ☀️🏃♂️💪",
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_SingleHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{"#### 23 May, Friday"}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{{"#### 23 May, Friday"}}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_MultipleConsecutiveHeaders(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 23 May, Friday",
|
||||
"#### 23 May, Saturday",
|
||||
"#### 24 May, Sunday",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{{
|
||||
"#### 23 May, Friday",
|
||||
"#### 23 May, Saturday",
|
||||
"#### 24 May, Sunday",
|
||||
}}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_HeadersWithEmojis(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 23 May, Friday 🍽💪",
|
||||
"#### 24 May, Saturday 💧",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{{
|
||||
"#### 23 May, Friday 🤸♂️",
|
||||
"#### 23 May, Friday 🍽💪",
|
||||
"#### 24 May, Saturday 💧",
|
||||
}}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_NonHeaders(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"This is not a header",
|
||||
"Neither is this",
|
||||
"Just regular text",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{
|
||||
{"This is not a header"},
|
||||
{"Neither is this"},
|
||||
{"Just regular text"},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_MixedHeadersAndNonHeaders(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 23 May, Friday",
|
||||
"#### 23 May, Saturday",
|
||||
"Some journal content",
|
||||
"More content",
|
||||
"#### 24 May, Sunday",
|
||||
"Final content",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{
|
||||
{"#### 23 May, Friday", "#### 23 May, Saturday"},
|
||||
{"Some journal content"},
|
||||
{"More content"},
|
||||
{"#### 24 May, Sunday"},
|
||||
{"Final content"},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_HeadersWithExtraSpaces(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Headers with extra spaces - should still match
|
||||
lines := []string{
|
||||
"#### 23 May, Friday ",
|
||||
"#### 23 May, Saturday", // Extra space after ####
|
||||
"#### 24 May, Sunday",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
// Only the properly formatted ones should be grouped as headers
|
||||
expected := [][]string{
|
||||
{"#### 23 May, Friday "},
|
||||
{"#### 23 May, Saturday"}, // This won't match the regex
|
||||
{"#### 24 May, Sunday"},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_InvalidHeaderFormats(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"### 23 May, Friday", // Wrong number of #
|
||||
"##### 23 May, Friday", // Too many #
|
||||
"#### May 23, Friday", // Wrong date format
|
||||
"#### 23 May Friday", // Missing comma
|
||||
"#### 23 May,", // Missing day
|
||||
"#### 23, Friday", // Missing month
|
||||
"#### twenty May, Friday", // Non-numeric day
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
// None should match, so each should be in its own group
|
||||
expected := [][]string{
|
||||
{"### 23 May, Friday"},
|
||||
{"##### 23 May, Friday"},
|
||||
{"#### May 23, Friday"},
|
||||
{"#### 23 May Friday"},
|
||||
{"#### 23 May,"},
|
||||
{"#### 23, Friday"},
|
||||
{"#### twenty May, Friday"},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_ValidHeaderVariations(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 1 Jan, Monday",
|
||||
"#### 10 February, Tuesday",
|
||||
"#### 31 December, Wednesday",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{{
|
||||
"#### 1 Jan, Monday",
|
||||
"#### 10 February, Tuesday",
|
||||
"#### 31 December, Wednesday",
|
||||
}}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_MultipleGroups(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 23 May, Friday",
|
||||
"#### 23 May, Saturday",
|
||||
"Some content between groups",
|
||||
"#### 24 May, Sunday",
|
||||
"#### 24 May, Monday",
|
||||
"More content",
|
||||
"#### 25 May, Tuesday",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{
|
||||
{"#### 23 May, Friday", "#### 23 May, Saturday"},
|
||||
{"Some content between groups"},
|
||||
{"#### 24 May, Sunday", "#### 24 May, Monday"},
|
||||
{"More content"},
|
||||
{"#### 25 May, Tuesday"},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_SingleNonHeader(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{"Just some text"}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{{"Just some text"}}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_HeadersAtStartAndEnd(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 23 May, Friday",
|
||||
"#### 23 May, Saturday",
|
||||
"Middle content",
|
||||
"#### 24 May, Sunday",
|
||||
"#### 24 May, Monday",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{
|
||||
{"#### 23 May, Friday", "#### 23 May, Saturday"},
|
||||
{"Middle content"},
|
||||
{"#### 24 May, Sunday", "#### 24 May, Monday"},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_HeadersWithDifferentContent(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
// Headers with additional content after the date/day
|
||||
lines := []string{
|
||||
"#### 23 May, Friday - Good day",
|
||||
"#### 23 May, Saturday 🌞 Sunny",
|
||||
"#### 24 May, Sunday (rainy)",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{{
|
||||
"#### 23 May, Friday - Good day",
|
||||
"#### 23 May, Saturday 🌞 Sunny",
|
||||
"#### 24 May, Sunday (rainy)",
|
||||
}}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_EdgeCaseWhitespace(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"", // Empty line
|
||||
"#### 23 May, Friday",
|
||||
" ", // Line with just spaces
|
||||
"#### 24 May, Saturday",
|
||||
"",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{
|
||||
{""},
|
||||
{"#### 23 May, Friday"},
|
||||
{" "},
|
||||
{"#### 24 May, Saturday"},
|
||||
{""},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_CaseSensitivity(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 23 may, friday", // lowercase
|
||||
"#### 23 MAY, FRIDAY", // uppercase
|
||||
"#### 23 May, Friday", // proper case
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
// Only the properly cased one should match
|
||||
expected := [][]string{
|
||||
{"#### 23 may, friday", "#### 23 MAY, FRIDAY", "#### 23 May, Friday"},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_RealWorldJournalExample(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 28 May, Wednesday",
|
||||
"#### 28 May, Wednesday", // Duplicate header
|
||||
"Morning routine:",
|
||||
"- Coffee ☕",
|
||||
"- Exercise 💪",
|
||||
"",
|
||||
"#### 29 May, Thursday",
|
||||
"#### 29 May, Thursday",
|
||||
"Work day:",
|
||||
"- Team meeting",
|
||||
"- Code review",
|
||||
"",
|
||||
"#### 30 May, Friday",
|
||||
"TGIF! 🎉",
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
expected := [][]string{
|
||||
{"#### 28 May, Wednesday", "#### 28 May, Wednesday"},
|
||||
{"Morning routine:"},
|
||||
{"- Coffee ☕"},
|
||||
{"- Exercise 💪"},
|
||||
{""},
|
||||
{"#### 29 May, Thursday", "#### 29 May, Thursday"},
|
||||
{"Work day:"},
|
||||
{"- Team meeting"},
|
||||
{"- Code review"},
|
||||
{""},
|
||||
{"#### 30 May, Friday"},
|
||||
{"TGIF! 🎉"},
|
||||
}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
|
||||
func TestGroupConsecutiveHeaders_RegexEdgeCases(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
lines := []string{
|
||||
"#### 0 May, Friday", // Zero day - should match \d+
|
||||
"#### 99 December, Sunday", // Large day number
|
||||
"#### 1 A, Monday", // Single letter month - should match \w+
|
||||
"#### 1 Ab, Tuesday", // Two letter month
|
||||
"#### 1 January, W", // Single letter day - should match \w+
|
||||
"#### 1 January, Wednesday", // Full day name
|
||||
}
|
||||
result := groupConsecutiveHeaders(lines)
|
||||
// All should match the regex pattern
|
||||
expected := [][]string{{
|
||||
"#### 0 May, Friday",
|
||||
"#### 99 December, Sunday",
|
||||
"#### 1 A, Monday",
|
||||
"#### 1 Ab, Tuesday",
|
||||
"#### 1 January, W",
|
||||
"#### 1 January, Wednesday",
|
||||
}}
|
||||
r.Equal(expected, result)
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
// TODO gzip
|
||||
package sync
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
const (
|
||||
StatusOK = "ok"
|
||||
StatusNotModified = "notModified"
|
||||
StatusUpdatedOnServer = "updatedOnServer"
|
||||
StatusMerged = "merged"
|
||||
|
||||
MaxTextSize = 5 << 20 // 5 MB
|
||||
MaxFilenamesSize = 10 << 20 // 10 MB
|
||||
MaxTokenSize = 4 << 10 // 4 KB
|
||||
)
|
||||
|
||||
var OnChatUpdate = func(userID int64) {}
|
||||
|
||||
type file struct {
|
||||
Status string `json:"status"`
|
||||
Path string `json:"path"`
|
||||
LastModified int64 `json:"lastModified"`
|
||||
ClientLastModified int64 `json:"clientLastModified,omitempty"`
|
||||
ClientLastSynced int64 `json:"clientLastSynced,omitempty"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type syncRequest struct {
|
||||
Modified []file `json:"modified"` // New or modified files from client
|
||||
Deleted []string `json:"deleted"` // Deleted files from client
|
||||
Timestamps map[string]int64 `json:"timestamps"`
|
||||
ServerTime int64 `json:"serverTime"` // latest server-event ts the client has acknowledged; server returns events newer than this
|
||||
}
|
||||
|
||||
type syncResponse struct {
|
||||
Status string `json:"status"` // Status
|
||||
Error string `json:"error,omitempty"` // Server-side reason for a 4xx/5xx so the client can log it
|
||||
Files []file `json:"files"` // Files with content that need syncing
|
||||
Timestamps map[string]int64 `json:"timestamps"` // Current server timestamps in Unix format
|
||||
Renames map[string]string `json:"renames"` // What files to rename on client
|
||||
Deleted map[string]int64 `json:"deleted"` // path -> deletedAt; client drops local copies older than this
|
||||
}
|
||||
|
||||
// SyncFilenames sync texts between client and server.
|
||||
// The following steps are executed:
|
||||
// 1) Save client-modified files to the server
|
||||
// 2) In case of conflict (server has a newer modification), merge the files and include them in the response
|
||||
// 3) Based on known client dirs timestamps, send newly updated or created files
|
||||
// 4) Respond with last modification timestamps for every dir
|
||||
func SyncFilenames(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.Method != http.MethodPost {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
_ = json.NewEncoder(w).Encode(syncResponse{Status: "error", Error: "Method not allowed"})
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, MaxFilenamesSize)
|
||||
|
||||
var request syncRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_ = json.NewEncoder(w).Encode(syncResponse{Status: "error", Error: fmt.Sprintf("Invalid syncFilenames JSON: %v", err)})
|
||||
return
|
||||
}
|
||||
|
||||
userFS, err := fs.NewUserFS(userID(r))
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncTexts: error creating user FS", "error", err)
|
||||
http.Error(w, "Error creating user FS", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Delete files.
|
||||
for _, path := range request.Deleted {
|
||||
// Paths that are coming from client start with /, make them relative
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
err = userFS.Del(fs.DirUserRoot, path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
slog.Error("Sync error: syncTexts: error deleting file", "path", path, "error", err)
|
||||
continue
|
||||
}
|
||||
logSync(fmt.Sprintf("❌ Sync texts: deleting file: '%s'", path), r)
|
||||
debugLogDelete(fmt.Sprintf("Deleting file: '%s'", path), r)
|
||||
}
|
||||
|
||||
// TODO using rename log first replace old paths in client request to new so other code will work okay
|
||||
// and maybe include it right away for files to send
|
||||
// TODO what if multiply moves, back and forth? Merge them?
|
||||
lastSync := int64(0)
|
||||
for _, ts := range request.Timestamps {
|
||||
if ts > lastSync {
|
||||
lastSync = ts
|
||||
}
|
||||
}
|
||||
// TODO if a file was changed on client on oldPath, merge it with the new path
|
||||
|
||||
renames := make(map[string]string)
|
||||
// Don't respond renames on first sync
|
||||
if lastSync != 0 {
|
||||
renames = RenamesLog(userID(r), lastSync)
|
||||
}
|
||||
|
||||
// Server-side events newer than the client's acknowledged watermark.
|
||||
deletes := DeletesLog(userID(r), request.ServerTime+1)
|
||||
|
||||
// Suppress echoes: don't return entries that THIS client just told us
|
||||
// about in request.Deleted. Match by suffix so it works regardless of
|
||||
// whether DeletesLog keys are stripped or absolute.
|
||||
for _, p := range request.Deleted {
|
||||
ownRel := strings.TrimPrefix(strings.TrimPrefix(p, "/"), "/")
|
||||
for key := range deletes {
|
||||
if key == ownRel || strings.HasSuffix(key, "/"+ownRel) {
|
||||
delete(deletes, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If a file was renamed and changed, on client we would rename then change?
|
||||
// Save client-modified files to the server
|
||||
for _, clientFile := range request.Modified {
|
||||
// Paths that are coming from client start with /, make them relative
|
||||
path := strings.TrimPrefix(clientFile.Path, "/")
|
||||
relativePath := strings.TrimPrefix(path, "/")
|
||||
|
||||
serverModifiedTime, err := userFS.Mtime(fs.DirUserRoot, relativePath)
|
||||
var clientContent string
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
slog.Error("Sync error: syncTexts: error reading file", "path", path, "error", err)
|
||||
logSync(fmt.Sprintf("Sync texts: error reading file '%s': %v", path, err), r)
|
||||
// TODO All-or-nothing sync?
|
||||
continue
|
||||
} else if errors.Is(err, os.ErrNotExist) {
|
||||
logSync(fmt.Sprintf("Sync texts: creating: '%s'", path), r)
|
||||
clientContent = clientFile.Content
|
||||
} else {
|
||||
// TODO file locks?
|
||||
fileWasModifiedOnServer := serverModifiedTime > clientFile.LastModified
|
||||
if fileWasModifiedOnServer {
|
||||
// Change on both client and server.
|
||||
serverContent, err := userFS.Read(fs.DirUserRoot, relativePath)
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncTexts: error reading modified on server file '%s': %v", path, err)
|
||||
continue
|
||||
}
|
||||
logSync(fmt.Sprintf("🔀 Sync texts: Merging and writing: '%s'", path), r)
|
||||
clientContent = Merge(serverContent, clientFile.Content)
|
||||
} else {
|
||||
// Changed on client, unchanged on server.
|
||||
logSync(fmt.Sprintf("💻 Sync texts: Writing only: '%s'", path), r)
|
||||
clientContent = clientFile.Content
|
||||
}
|
||||
}
|
||||
|
||||
// We don't accept config from client, because for now it is only modified on server.
|
||||
// Plus we need to mess with JSON merging :)
|
||||
if clientFile.Path == config.ServerCfg.ConfigFilename {
|
||||
continue
|
||||
}
|
||||
|
||||
// Write the clientContent to the server at path.
|
||||
err = userFS.Write(fs.DirUserRoot, relativePath, clientContent)
|
||||
if errors.Is(err, fs.ErrQuotaExceeded) {
|
||||
http.Error(w, `{"error":"Storage quota exceeded"}`, http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncTexts: error writing file '%s': %v", path, err)
|
||||
logSync(fmt.Sprintf("Sync texts: error writing file '%s': %v", path, err), r)
|
||||
continue
|
||||
}
|
||||
|
||||
if relativePath == fs.ChatFilename {
|
||||
OnChatUpdate(userID(r))
|
||||
}
|
||||
}
|
||||
|
||||
// Based on known client dirs timestamps, send newly updated or created files.
|
||||
serverTimestamps, err := userFS.Mtimes(fs.DirUserRoot, fs.MDExt, ".txt")
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncTexts: error getting server timestamps", "error", err)
|
||||
http.Error(w, fmt.Sprintf("Failed to get timestamps: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Include config file timestamp, so it will be sent to the client if stale.
|
||||
configCtime, err := userFS.Mtime(fs.DirUserRoot, config.ServerCfg.ConfigFilename)
|
||||
// We can ignore the error since config.json is not used on client in any way, pure for read-only purposes.
|
||||
if err == nil {
|
||||
serverTimestamps[config.ServerCfg.ConfigFilename] = configCtime
|
||||
}
|
||||
|
||||
// Prepare the list of files to send to the client
|
||||
// TODO optimize don't send files known to client.
|
||||
// For now we save client file to server, and the code below would include it again.
|
||||
files := make([]file, 0)
|
||||
dirTimestamps := make(map[string]int64)
|
||||
for path, serverFileTime := range serverTimestamps {
|
||||
// TOOD make it not as ugly?
|
||||
parts := strings.Split(path, string(os.PathSeparator))
|
||||
dir := parts[0]
|
||||
isInRoot := len(parts) == 1
|
||||
if isInRoot {
|
||||
dir = "."
|
||||
}
|
||||
|
||||
requestDirTime, exists := request.Timestamps[dir]
|
||||
if !exists || serverFileTime > requestDirTime {
|
||||
// Client needs this file - read its content
|
||||
content, err := userFS.Read(fs.DirUserRoot, path)
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncTexts: error reading file", "path", path, "error", err)
|
||||
logSync(fmt.Sprintf("Error reading file %s: %v", path, err), r)
|
||||
continue
|
||||
}
|
||||
|
||||
files = append(files, file{
|
||||
Status: StatusOK,
|
||||
Path: path,
|
||||
LastModified: serverFileTime,
|
||||
Content: content,
|
||||
})
|
||||
}
|
||||
|
||||
// Calculate the latest file timestamp for each directory
|
||||
existingTimestamp, exists := dirTimestamps[dir]
|
||||
if !exists {
|
||||
dirTimestamps[dir] = serverFileTime
|
||||
continue
|
||||
}
|
||||
if serverFileTime > existingTimestamp {
|
||||
dirTimestamps[dir] = serverFileTime
|
||||
}
|
||||
}
|
||||
|
||||
// TODO Calculate deletions for client (files that exist on client but not on server)
|
||||
|
||||
response := syncResponse{
|
||||
Status: StatusOK,
|
||||
Files: files,
|
||||
Timestamps: dirTimestamps,
|
||||
Renames: renames,
|
||||
Deleted: deletes,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
http.Error(w, "Error encoding response", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
func SyncFile(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, MaxTextSize)
|
||||
|
||||
var clientFile file
|
||||
if err := json.NewDecoder(r.Body).Decode(&clientFile); err != nil {
|
||||
http.Error(w, "Invalid syncMediasRequest JSON", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
userFS, err := fs.NewUserFS(userID(r))
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncText: error creating user FS", "error", err)
|
||||
http.Error(w, "Error creating user FS", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// 1) Save client-modified file to the server
|
||||
// 2) In case of conflict (server has a newer modification), merge the clientFile and include them in the response
|
||||
|
||||
// Paths that are coming from client start with /, make them relative.
|
||||
path := clientFile.Path
|
||||
relativePath := strings.TrimPrefix(path, "/")
|
||||
|
||||
// TODO if no clientFile, severContent = ""
|
||||
serverContent, err := userFS.Read(fs.DirUserRoot, relativePath)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
slog.Error("Sync error: syncText: error reading clientFile", "path", path, "error", err)
|
||||
http.Error(w, "Error reading server clientFile", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
serverLastModified, err := userFS.Mtime(fs.DirUserRoot, relativePath)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
slog.Error("Sync error: syncText: error getting ctime for clientFile '%s': %v", path, err)
|
||||
http.Error(w, "Error getting ctime for clientFile", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// TODO when clientFile does not exist the content is empty, which is implicit
|
||||
// Return already up-to-date status
|
||||
if serverContent == clientFile.Content {
|
||||
response := map[string]interface{}{
|
||||
"status": StatusNotModified,
|
||||
"lastModified": serverLastModified,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(response)
|
||||
return
|
||||
}
|
||||
|
||||
logSync(fmt.Sprintf("Client file '%s': last client modified: %d, last client synced: %d", path, clientFile.ClientLastModified, clientFile.ClientLastSynced), r)
|
||||
|
||||
status := StatusOK
|
||||
var content string
|
||||
fileWasModifiedOnServer := false
|
||||
shouldUpdateOnServer := true
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
logSync(fmt.Sprintf("Creating one clientFile: '%s'", path), r)
|
||||
content = clientFile.Content
|
||||
} else {
|
||||
wasNotModifiedOnClient := clientFile.ClientLastSynced != 0 && clientFile.ClientLastModified == clientFile.ClientLastSynced
|
||||
fileWasModifiedOnServer = serverLastModified > clientFile.LastModified
|
||||
if fileWasModifiedOnServer && wasNotModifiedOnClient {
|
||||
logSync(fmt.Sprintf("📡 Modified only on server, sending server copy to client: '%s'", path), r)
|
||||
content = serverContent
|
||||
shouldUpdateOnServer = false
|
||||
} else if fileWasModifiedOnServer { // Modified on both server and client
|
||||
logSync(fmt.Sprintf("File '%s' was modified on server at %d, but on client at %d", path, serverLastModified, clientFile.ClientLastModified), r)
|
||||
logSync(fmt.Sprintf("🔀 Merging and writing one clientFile: '%s'", path), r)
|
||||
content = Merge(serverContent, clientFile.Content)
|
||||
status = StatusMerged
|
||||
} else {
|
||||
// TODO for resilience add merge here, because we had case when server saved latest TS but no conent.
|
||||
// Also, if for some reason timestamps would change on server migration and such.
|
||||
// Server clientFile hasn't changed since client's last sync
|
||||
logSync(fmt.Sprintf("💻 Modified only on client, writing to server: '%s'", path), r)
|
||||
content = clientFile.Content
|
||||
}
|
||||
}
|
||||
|
||||
if shouldUpdateOnServer {
|
||||
err = userFS.Write(fs.DirUserRoot, relativePath, content)
|
||||
if errors.Is(err, fs.ErrQuotaExceeded) {
|
||||
http.Error(w, `{"error":"Storage quota exceeded"}`, http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncText: error writing clientFile '%s': %v", path, err)
|
||||
logSync(fmt.Sprintf("Error writing clientFile '%s': %v", path, err), r)
|
||||
http.Error(w, "Error writing clientFile", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if relativePath == fs.ChatFilename {
|
||||
OnChatUpdate(userID(r))
|
||||
}
|
||||
}
|
||||
|
||||
serverLastModified, err = userFS.Mtime(fs.DirUserRoot, relativePath)
|
||||
// TODO what if 0?
|
||||
logSync(fmt.Sprintf("Final server timestamp for '%s': %d", path, serverLastModified), r)
|
||||
|
||||
if !fileWasModifiedOnServer {
|
||||
response := map[string]interface{}{
|
||||
"status": StatusUpdatedOnServer,
|
||||
"lastModified": serverLastModified,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(response)
|
||||
return
|
||||
}
|
||||
|
||||
response := file{
|
||||
Status: status,
|
||||
Content: content,
|
||||
Path: path,
|
||||
LastModified: serverLastModified,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
http.Error(w, "Error encoding response", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func logSync(msg string, r *http.Request) {
|
||||
msg = fmt.Sprintf("%d: %s", userID(r), msg)
|
||||
|
||||
file, err := os.OpenFile("/tmp/sync", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
fmt.Println("Error opening log file:", err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if version := r.Header.Get("Version"); version != "" {
|
||||
msg = fmt.Sprintf("%s (version: %s)", msg, version)
|
||||
} else {
|
||||
msg = fmt.Sprintf("%s (version: unknown)", msg)
|
||||
}
|
||||
time := time.Now().Format("2006-01-02 15:04:05")
|
||||
msg = fmt.Sprintf("%s: %s\n", time, msg)
|
||||
if _, err := file.WriteString(msg); err != nil {
|
||||
slog.Error("Sync error: logSync: error writing to log file", "error", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func debugLogDelete(msg string, r *http.Request) {
|
||||
msg = fmt.Sprintf("%d: %s", userID(r), msg)
|
||||
file, err := os.OpenFile("/tmp/del", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
slog.Error("Sync error: logDelete: error opening log file", "error", err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
time := time.Now().Format("2006-01-02 15:04:05")
|
||||
if _, err := file.WriteString(time + ": " + msg + "\n"); err != nil {
|
||||
fmt.Println("Error writing to log file:", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func userID(r *http.Request) int64 {
|
||||
return r.Context().Value("userID").(int64)
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxMediaSize = 65 << 30 // 65 MB
|
||||
MaxMediaFilenamesSize = 512 << 10 // 512 KB
|
||||
)
|
||||
|
||||
var syncMediasRequest struct {
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
FilenamesHash string `json:"filenamesHash"`
|
||||
}
|
||||
|
||||
type media struct {
|
||||
Filename string `json:"filename"`
|
||||
LastModified int64 `json:"lastModified"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
func SyncMediaFilenames(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, MaxMediaFilenamesSize)
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&syncMediasRequest); err != nil {
|
||||
http.Error(w, "Invalid syncMediasRequest JSON", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
userFS, err := fs.NewUserFS(userID(r))
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncMedias: error creating media FS", "error", err)
|
||||
http.Error(w, "Error creating media FS", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Find media files newer than client's timestamp
|
||||
ctimes, err := userFS.Mtimes(fs.DirMedia)
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncMedias: error getting media file times", "error", err)
|
||||
http.Error(w, "Error getting media file times", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
mediaFiles := make([]media, 0)
|
||||
latestTimestamp := int64(0)
|
||||
for filename, modTime := range ctimes {
|
||||
// TODO theoretically it is possible to miss some files if there were created in the same second.
|
||||
if modTime <= syncMediasRequest.Timestamp {
|
||||
continue
|
||||
}
|
||||
if modTime > latestTimestamp {
|
||||
latestTimestamp = modTime
|
||||
}
|
||||
|
||||
mediaFiles = append(mediaFiles, media{
|
||||
Filename: filename,
|
||||
LastModified: modTime,
|
||||
})
|
||||
}
|
||||
|
||||
response := struct {
|
||||
Files []media `json:"files"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}{
|
||||
Files: mediaFiles,
|
||||
Timestamp: latestTimestamp,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
http.Error(w, "Error encoding response", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
// SyncMediaFile syncs a single media file by path.
|
||||
func SyncMediaFile(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, MaxMediaSize)
|
||||
|
||||
var clientMedia media
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("Cannot read request body: %s", err), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer r.Body.Close()
|
||||
|
||||
if err := json.Unmarshal(body, &clientMedia); err != nil {
|
||||
http.Error(w, fmt.Sprintf("Invalid syncMedia Request JSON: %s", body), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
userFS, err := fs.NewUserFS(userID(r))
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncMedia: error creating user FS", "error", err)
|
||||
http.Error(w, "Error creating user FS", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
exists, err := userFS.Exists(fs.DirMedia, clientMedia.Filename)
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncMedia: error checking media existence", "error", err)
|
||||
http.Error(w, "Error checking media existence", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
shouldWriteToServer := clientMedia.Data != "" && !exists
|
||||
if shouldWriteToServer {
|
||||
content, err := base64.StdEncoding.DecodeString(clientMedia.Data)
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid base64 data", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
err = userFS.Write(fs.DirMedia, clientMedia.Filename, string(content))
|
||||
if errors.Is(err, fs.ErrQuotaExceeded) {
|
||||
http.Error(w, `{"error":"Storage quota exceeded"}`, http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid base64 data", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
logSync(fmt.Sprintf("Media created: %s", clientMedia.Filename), r)
|
||||
return
|
||||
}
|
||||
|
||||
path, err := userFS.SafePath(fs.DirMedia, clientMedia.Filename)
|
||||
if err != nil {
|
||||
slog.Error("Sync error: syncMedia: unsafe path", "error", err)
|
||||
http.Error(w, "The path is unsafe", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
@@ -0,0 +1,514 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
func init() {
|
||||
fs.Ctime = func(fi os.FileInfo) int64 {
|
||||
return 1
|
||||
}
|
||||
fs.Mtime = func(fi os.FileInfo) int64 {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncText_CreateNewFileOnServer(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return fs.NewFS("/", afero.NewMemMapFs())
|
||||
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
clientFile := file{
|
||||
Path: "test.md",
|
||||
Content: "Hello World",
|
||||
LastModified: 1234567890,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(clientFile)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/syncText", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFile(w, req)
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response file
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal(StatusUpdatedOnServer, response.Status)
|
||||
r.True(response.LastModified > 0)
|
||||
}
|
||||
|
||||
func TestSyncText_UpdateExistingFile_NoConflict(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return userFS, nil
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
err = userFS.Write("", "test.md", "Original content")
|
||||
r.NoError(err)
|
||||
|
||||
clientFile := file{
|
||||
Path: "test.md",
|
||||
Content: "Updated content",
|
||||
LastModified: 1,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(clientFile)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/syncText", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFile(w, req)
|
||||
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response map[string]interface{}
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal(StatusUpdatedOnServer, response["status"])
|
||||
|
||||
// Verify content was updated
|
||||
content, err := userFS.Read("", "test.md")
|
||||
r.NoError(err)
|
||||
r.Equal("Updated content", content)
|
||||
}
|
||||
|
||||
func TestSyncText_NotModified(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return userFS, nil
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
err = userFS.Write("", "test.md", "Original content")
|
||||
r.NoError(err)
|
||||
|
||||
clientFile := file{
|
||||
Path: "test.md",
|
||||
Content: "Original content",
|
||||
LastModified: 1,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(clientFile)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/syncText", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFile(w, req)
|
||||
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response map[string]interface{}
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal(StatusNotModified, response["status"])
|
||||
|
||||
// Verify content was updated
|
||||
content, err := userFS.Read("", "test.md")
|
||||
r.NoError(err)
|
||||
r.Equal("Original content", content)
|
||||
}
|
||||
|
||||
func TestSyncText_UpdateExistingFile_Conflict(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return userFS, nil
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
err = userFS.Write("", "test.md", "Server content")
|
||||
r.NoError(err)
|
||||
|
||||
clientFile := file{
|
||||
Path: "test.md",
|
||||
Content: "Client content",
|
||||
LastModified: 0,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(clientFile)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/syncText", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFile(w, req)
|
||||
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response map[string]interface{}
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal("merged", response["status"])
|
||||
|
||||
// Verify content was updated
|
||||
content, err := userFS.Read("", "test.md")
|
||||
r.NoError(err)
|
||||
r.Equal("Server content\nClient content", content)
|
||||
}
|
||||
|
||||
func TestSyncText_UpdateExistingFile_JournalConflict(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return userFS, nil
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
err = userFS.Write("", "test.md", "#### 25 May, Friday 🚀\nServer content")
|
||||
r.NoError(err)
|
||||
|
||||
clientFile := file{
|
||||
Path: "test.md",
|
||||
Content: "#### 25 May, Friday\nServer content",
|
||||
LastModified: 0,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(clientFile)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/syncText", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFile(w, req)
|
||||
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response map[string]interface{}
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal("merged", response["status"])
|
||||
|
||||
// Verify content was updated
|
||||
content, err := userFS.Read("", "test.md")
|
||||
r.NoError(err)
|
||||
r.Equal("#### 25 May, Friday 🚀\nServer content", content)
|
||||
}
|
||||
|
||||
func TestSyncAllTexts_EmptyRequest(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
origFilename := config.ServerCfg.ConfigFilename
|
||||
config.ServerCfg.ConfigFilename = "config.json"
|
||||
defer func() {
|
||||
config.ServerCfg.ConfigFilename = origFilename
|
||||
}()
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return userFS, nil
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
request := syncRequest{
|
||||
Timestamps: make(map[string]int64),
|
||||
Modified: []file{},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(request)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/syncTexts", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFilenames(w, req)
|
||||
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response syncResponse
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal(StatusOK, response.Status)
|
||||
r.Empty(response.Files)
|
||||
r.Empty(response.Renames)
|
||||
}
|
||||
|
||||
func TestSyncAllTexts_CreateNewFilesOnServer(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
origFilename := config.ServerCfg.ConfigFilename
|
||||
config.ServerCfg.ConfigFilename = "config.json"
|
||||
defer func() {
|
||||
config.ServerCfg.ConfigFilename = origFilename
|
||||
}()
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return userFS, nil
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
request := syncRequest{
|
||||
Timestamps: make(map[string]int64),
|
||||
Modified: []file{
|
||||
{
|
||||
Path: "today/task1.md",
|
||||
Content: "Task 1 content",
|
||||
LastModified: 0,
|
||||
},
|
||||
{
|
||||
Path: "later/task2.md",
|
||||
Content: "Task 2 content",
|
||||
LastModified: 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(request)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/syncTexts", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFilenames(w, req)
|
||||
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response syncResponse
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal(StatusOK, response.Status)
|
||||
r.Len(response.Files, 2)
|
||||
r.Contains(response.Timestamps, "today")
|
||||
r.Contains(response.Timestamps, "later")
|
||||
|
||||
// Verify files were created on server
|
||||
r.NoError(err)
|
||||
content1, err := userFS.Read("today", "task1.md")
|
||||
r.NoError(err)
|
||||
r.Equal("Task 1 content", content1)
|
||||
|
||||
content2, err := userFS.Read("later", "task2.md")
|
||||
r.NoError(err)
|
||||
r.Equal("Task 2 content", content2)
|
||||
}
|
||||
|
||||
func TestSyncAllTexts_UpdateExistingFilesOnServer(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return userFS, nil
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
err = userFS.Write("", "today/existing.md", "Original content")
|
||||
r.NoError(err)
|
||||
|
||||
request := syncRequest{
|
||||
Timestamps: map[string]int64{
|
||||
"today": 0, // Old timestamp
|
||||
},
|
||||
Modified: []file{
|
||||
{
|
||||
Path: "today/existing.md",
|
||||
Content: "Updated content",
|
||||
LastModified: 1, // Same as server time
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(request)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/sync-all", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFilenames(w, req)
|
||||
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response syncResponse
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal(StatusOK, response.Status)
|
||||
|
||||
// Verify file was updated
|
||||
content, err := userFS.Read("", "today/existing.md")
|
||||
r.NoError(err)
|
||||
r.Equal("Updated content", content)
|
||||
}
|
||||
|
||||
func TestSyncAllTexts_SendUpdatedFilesToClient(t *testing.T) {
|
||||
r := require.New(t)
|
||||
|
||||
origFilename := config.ServerCfg.ConfigFilename
|
||||
config.ServerCfg.ConfigFilename = "config.json"
|
||||
defer func() {
|
||||
config.ServerCfg.ConfigFilename = origFilename
|
||||
}()
|
||||
|
||||
userFS, err := fs.NewFS("/", afero.NewMemMapFs())
|
||||
r.NoError(err)
|
||||
origFS := fs.NewUserFS
|
||||
fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
return userFS, nil
|
||||
}
|
||||
defer func() {
|
||||
fs.NewUserFS = origFS
|
||||
}()
|
||||
|
||||
// Create files on server
|
||||
err = userFS.Write("", "today/new.md", "New server file")
|
||||
r.NoError(err)
|
||||
err = userFS.Write("", "today/old.md", "Old file")
|
||||
r.NoError(err)
|
||||
|
||||
request := syncRequest{
|
||||
Timestamps: map[string]int64{
|
||||
"today": 0,
|
||||
},
|
||||
Modified: []file{},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(request)
|
||||
r.NoError(err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/syncTexts", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req = req.WithContext(context.WithValue(req.Context(), "userID", int64(-1)))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
SyncFilenames(w, req)
|
||||
|
||||
r.Equal(http.StatusOK, w.Code)
|
||||
|
||||
var response syncResponse
|
||||
err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
r.NoError(err)
|
||||
r.Equal(StatusOK, response.Status)
|
||||
r.Len(response.Files, 2) // Both files should be sent
|
||||
|
||||
// Check that files contain content
|
||||
for _, file := range response.Files {
|
||||
r.NotEmpty(file.Content)
|
||||
r.True(file.LastModified > 0)
|
||||
}
|
||||
}
|
||||
|
||||
//func TestSyncAllTexts_PathTraversalAttack(t *testing.T) {
|
||||
// r := require.New(t)
|
||||
//
|
||||
// memFS := afero.NewMemMapFs()
|
||||
// rootFS, err := fs.NewFS("/", memFS)
|
||||
// r.NoError(err)
|
||||
// err = rootFS.Write("", "user1/secret1.md", "New server file")
|
||||
// r.NoError(err)
|
||||
// err = rootFS.Write("", "user2/secret2.md", "Old file")
|
||||
// r.NoError(err)
|
||||
//
|
||||
// userFS, err := fs.NewFS("/user1", afero.NewMemMapFs())
|
||||
// r.NoError(err)
|
||||
// origFS := fs.NewUserFS
|
||||
// fs.NewUserFS = func(userID int64) (*fs.FS, error) {
|
||||
// return userFS, nil
|
||||
// }
|
||||
// defer func() {
|
||||
// fs.NewUserFS = origFS
|
||||
// }()
|
||||
//
|
||||
// request := syncRequest{
|
||||
// UserID: 0,
|
||||
// Timestamps: map[string]int64{
|
||||
// "": 0,
|
||||
// },
|
||||
// Modified: []file{},
|
||||
// }
|
||||
//
|
||||
// body, err := json.Marshal(request)
|
||||
// r.NoError(err)
|
||||
//
|
||||
// req := httptest.NewRequest(http.MethodPost, "/syncTexts", bytes.NewBuffer(body))
|
||||
// req.DisplayName.Set("Content-Type", "application/json")
|
||||
// w := httptest.NewRecorder()
|
||||
//
|
||||
// SyncTexts(w, req)
|
||||
//
|
||||
// r.Equal(http.StatusOK, w.Code)
|
||||
//
|
||||
// var response syncResponse
|
||||
// err = json.Unmarshal(w.Body.Bytes(), &response)
|
||||
// r.NoError(err)
|
||||
// r.Equal(StatusOK, response.Status)
|
||||
// r.Len(response.Modified, 2) // Both files should be sent
|
||||
//
|
||||
// // Check that files contain content
|
||||
// for _, file := range response.Modified {
|
||||
// r.NotEmpty(file.Content)
|
||||
// r.True(file.LastModified > 0)
|
||||
// }
|
||||
//}
|
||||
@@ -0,0 +1,286 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
)
|
||||
|
||||
const (
|
||||
TokenLength = 32
|
||||
OneTimeTokenExpiration = 10 * time.Minute
|
||||
BanForInvalidToken = 10 * time.Minute
|
||||
AuthCookieName = "token"
|
||||
AuthCookieMaxAge = 10 * 365 * 24 * 60 * 60 // ~10 years
|
||||
)
|
||||
|
||||
var (
|
||||
oneTimeTokens = make(map[string]oneTimeToken)
|
||||
mu sync.RWMutex
|
||||
)
|
||||
|
||||
var blockedIPs = make(map[string]time.Time)
|
||||
var blockedIPsMutex sync.RWMutex
|
||||
|
||||
type oneTimeToken struct {
|
||||
userID int64
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
func GenOneTimeToken(userID int64) string {
|
||||
token := genToken()
|
||||
|
||||
mu.Lock()
|
||||
oneTimeTokens[token] = oneTimeToken{
|
||||
userID: userID,
|
||||
expiresAt: time.Now().Add(OneTimeTokenExpiration),
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
return token
|
||||
}
|
||||
|
||||
func findUserID(token string) (int64, bool) {
|
||||
tokens, err := fs.NewFS(config.ServerCfg.TokensDir, afero.NewOsFs())
|
||||
if err != nil {
|
||||
slog.Error("Failed to create file system for tokens", "error", err)
|
||||
return 0, false
|
||||
}
|
||||
|
||||
data, err := tokens.Read("/", hashToken(token))
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
userID, err := strconv.ParseInt(data, 10, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return userID, true
|
||||
}
|
||||
|
||||
func setAuthCookie(w http.ResponseWriter, token string) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: AuthCookieName,
|
||||
Value: token,
|
||||
Path: "/",
|
||||
MaxAge: AuthCookieMaxAge,
|
||||
HttpOnly: true,
|
||||
Secure: true,
|
||||
SameSite: http.SameSiteNoneMode,
|
||||
})
|
||||
}
|
||||
|
||||
func IssueToken(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("PANIC in IssueToken: %v", r)
|
||||
http.Error(w, "Internal server error", 500)
|
||||
}
|
||||
}()
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
permanentToken, ok := issueNewPermanentToken(r)
|
||||
if !ok {
|
||||
// issueNewPermanentToken already logged the precise sub-reason.
|
||||
http.Error(w, "Invalid or expired token", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
setAuthCookie(w, permanentToken)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err := json.NewEncoder(w).Encode(map[string]string{"token": permanentToken})
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// TODO CHECK that user id belongs to oneTimeToken ID, or get user id by oneTimeToken
|
||||
// TODO add tests
|
||||
// TODO too harsh blocking, we may need to take into account proxies
|
||||
func tokenMiddleware(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
ip := getIPFromRemoteAddr(r.RemoteAddr)
|
||||
|
||||
blockedIPsMutex.RLock()
|
||||
blockedUntil, isBlocked := blockedIPs[ip]
|
||||
blockedIPsMutex.RUnlock()
|
||||
if isBlocked && time.Now().Before(blockedUntil) {
|
||||
// 429s are too noisy to log — a blocked IP can replay them every
|
||||
// few seconds. The originating 401 has already been recorded.
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
|
||||
// Try cookie first, fall back to Authorization header.
|
||||
var token string
|
||||
fromCookie := false
|
||||
if cookie, err := r.Cookie(AuthCookieName); err == nil && cookie.Value != "" {
|
||||
token = cookie.Value
|
||||
fromCookie = true
|
||||
}
|
||||
if token == "" {
|
||||
token = r.Header.Get("Authorization")
|
||||
}
|
||||
|
||||
userID, ok := findUserID(token)
|
||||
if !ok {
|
||||
blockedIPsMutex.Lock()
|
||||
blockedIPs[ip] = time.Now().Add(BanForInvalidToken)
|
||||
blockedIPsMutex.Unlock()
|
||||
|
||||
logAuthFailure("middleware_invalid_token_401", r, map[string]any{
|
||||
"http_status": 401,
|
||||
"token_source": map[bool]string{true: "cookie", false: "auth_header"}[fromCookie],
|
||||
"token_empty": token == "",
|
||||
"new_block_until": time.Now().Add(BanForInvalidToken).Format(time.RFC3339Nano),
|
||||
"new_block_for": BanForInvalidToken.String(),
|
||||
})
|
||||
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
// Migrate old clients from Authorization header to cookie.
|
||||
if !fromCookie {
|
||||
setAuthCookie(w, token)
|
||||
}
|
||||
|
||||
ctx := context.WithValue(r.Context(), "userID", userID)
|
||||
next(w, r.WithContext(ctx))
|
||||
}
|
||||
}
|
||||
|
||||
// TODO add tests
|
||||
func issueNewPermanentToken(r *http.Request) (string, bool) {
|
||||
r.Body = http.MaxBytesReader(nil, r.Body, MaxTokenSize)
|
||||
|
||||
var req struct {
|
||||
OneTimeToken string `json:"oneTimeToken"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
logAuthFailure("onetime_swap_body_decode_error_401", r, map[string]any{
|
||||
"http_status": 401,
|
||||
"decode_err": err.Error(),
|
||||
})
|
||||
return "", false
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
data, exists := oneTimeTokens[req.OneTimeToken]
|
||||
if !exists || time.Now().After(data.expiresAt) {
|
||||
// Snapshot a few one-time tokens for cross-referencing — fingerprints
|
||||
// only, never any portion of the raw token (still-live secrets).
|
||||
samplePrefixes := make([]string, 0, 5)
|
||||
now := time.Now()
|
||||
var liveCount, expiredCount int
|
||||
for tok, d := range oneTimeTokens {
|
||||
if now.After(d.expiresAt) {
|
||||
expiredCount++
|
||||
} else {
|
||||
liveCount++
|
||||
}
|
||||
if len(samplePrefixes) < 5 {
|
||||
samplePrefixes = append(samplePrefixes,
|
||||
fmt.Sprintf("%s(uid=%d,exp=%s)", tokenFingerprint(tok), d.userID, d.expiresAt.Format(time.RFC3339)))
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
reason := "onetime_token_not_in_map_401"
|
||||
if exists {
|
||||
reason = "onetime_token_expired_401"
|
||||
}
|
||||
extras := map[string]any{
|
||||
"http_status": 401,
|
||||
"submitted_token": tokenFingerprint(req.OneTimeToken),
|
||||
"submitted_token_blank": req.OneTimeToken == "",
|
||||
"map_live_count": liveCount,
|
||||
"map_expired_count": expiredCount,
|
||||
"map_sample": strings.Join(samplePrefixes, ","),
|
||||
"new_block_for": "1m",
|
||||
}
|
||||
if exists {
|
||||
extras["matched_user_id"] = data.userID
|
||||
extras["matched_expires_at"] = data.expiresAt.Format(time.RFC3339Nano)
|
||||
extras["matched_age"] = time.Since(data.expiresAt.Add(-OneTimeTokenExpiration)).String()
|
||||
}
|
||||
logAuthFailure(reason, r, extras)
|
||||
|
||||
return "", false
|
||||
}
|
||||
delete(oneTimeTokens, req.OneTimeToken)
|
||||
mu.Unlock()
|
||||
|
||||
token := genToken()
|
||||
tokens, err := fs.NewFS(config.ServerCfg.TokensDir, afero.NewOsFs())
|
||||
if err != nil {
|
||||
slog.Error("Failed to create file system for tokens", "error", err)
|
||||
logAuthFailure("onetime_swap_fs_init_error_401", r, map[string]any{
|
||||
"http_status": 401,
|
||||
"err": err.Error(),
|
||||
"user_id": data.userID,
|
||||
})
|
||||
return "", false
|
||||
}
|
||||
err = tokens.Write(fs.DirUserRoot, hashToken(token), strconv.FormatInt(data.userID, 10))
|
||||
if err != nil {
|
||||
logAuthFailure("onetime_swap_write_error_401", r, map[string]any{
|
||||
"http_status": 401,
|
||||
"err": err.Error(),
|
||||
"user_id": data.userID,
|
||||
})
|
||||
return "", false
|
||||
}
|
||||
|
||||
return token, true
|
||||
}
|
||||
|
||||
func genToken() string {
|
||||
bytes := make([]byte, TokenLength)
|
||||
rand.Read(bytes)
|
||||
return hex.EncodeToString(bytes)
|
||||
}
|
||||
|
||||
func hashToken(token string) string {
|
||||
// A token is a server-generated 32 bytes of entropy, so SHA-256 is fine here.
|
||||
// At 1 billion SHA256 hashes per second it would take ~10^60 years to brute force.
|
||||
h := sha256.New()
|
||||
h.Write([]byte(token + config.ServerCfg.TokensSalt))
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
func getIPFromRemoteAddr(remoteAddr string) string {
|
||||
host, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err != nil {
|
||||
// If SplitHostPort fails, might be just an IP without port
|
||||
if ip := net.ParseIP(remoteAddr); ip != nil {
|
||||
return remoteAddr
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
return host
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Debug instrumentation: append every auth failure (invalid one-time token
|
||||
// swap, invalid permanent token, IP-blocked refusals) to /tmp/auth so we can
|
||||
// diagnose the "invalid token after server runs a while" report.
|
||||
const authDbgPath = "/tmp/auth"
|
||||
|
||||
var authDbgMu sync.Mutex
|
||||
|
||||
// Never log any portion of the raw token: even an 8-char prefix is enough
|
||||
// to drastically narrow brute-force, and the value here may be a still-live
|
||||
// secret. Hash-only fingerprint is enough to correlate occurrences.
|
||||
func tokenFingerprint(t string) string {
|
||||
if t == "" {
|
||||
return "<empty>"
|
||||
}
|
||||
h := sha256.Sum256([]byte(t))
|
||||
return fmt.Sprintf("len=%d sha256_8=%s", len(t), hex.EncodeToString(h[:])[:8])
|
||||
}
|
||||
|
||||
func logAuthFailure(reason string, r *http.Request, extras map[string]any) {
|
||||
cookieVal := ""
|
||||
cookiePresent := false
|
||||
if c, err := r.Cookie(AuthCookieName); err == nil {
|
||||
cookieVal = c.Value
|
||||
cookiePresent = true
|
||||
}
|
||||
authHeader := r.Header.Get("Authorization")
|
||||
|
||||
mu.RLock()
|
||||
onetimeMapSize := len(oneTimeTokens)
|
||||
mu.RUnlock()
|
||||
|
||||
blockedIPsMutex.RLock()
|
||||
blockedMapSize := len(blockedIPs)
|
||||
blockedUntil, ipBlocked := blockedIPs[getIPFromRemoteAddr(r.RemoteAddr)]
|
||||
blockedIPsMutex.RUnlock()
|
||||
|
||||
parts := []string{
|
||||
"ts=" + time.Now().Format(time.RFC3339Nano),
|
||||
"reason=" + reason,
|
||||
"method=" + r.Method,
|
||||
"path=" + r.URL.Path,
|
||||
"remote=" + r.RemoteAddr,
|
||||
"ip=" + getIPFromRemoteAddr(r.RemoteAddr),
|
||||
"ua=" + strconv.Quote(r.UserAgent()),
|
||||
"x_forwarded_for=" + strconv.Quote(r.Header.Get("X-Forwarded-For")),
|
||||
"x_real_ip=" + strconv.Quote(r.Header.Get("X-Real-IP")),
|
||||
"cf_connecting_ip=" + strconv.Quote(r.Header.Get("CF-Connecting-IP")),
|
||||
"referer=" + strconv.Quote(r.Referer()),
|
||||
"cookie_present=" + strconv.FormatBool(cookiePresent),
|
||||
"cookie=" + tokenFingerprint(cookieVal),
|
||||
"auth_header_present=" + strconv.FormatBool(authHeader != ""),
|
||||
"auth_header=" + tokenFingerprint(authHeader),
|
||||
"onetime_map_size=" + strconv.Itoa(onetimeMapSize),
|
||||
"blocked_map_size=" + strconv.Itoa(blockedMapSize),
|
||||
"ip_currently_blocked=" + strconv.FormatBool(ipBlocked && time.Now().Before(blockedUntil)),
|
||||
}
|
||||
if ipBlocked {
|
||||
parts = append(parts,
|
||||
"ip_block_until="+blockedUntil.Format(time.RFC3339Nano),
|
||||
"ip_block_remaining="+time.Until(blockedUntil).String(),
|
||||
)
|
||||
}
|
||||
for k, v := range extras {
|
||||
parts = append(parts, fmt.Sprintf("%s=%v", k, v))
|
||||
}
|
||||
|
||||
line := strings.Join(parts, " ") + "\n"
|
||||
authDbgMu.Lock()
|
||||
defer authDbgMu.Unlock()
|
||||
f, err := os.OpenFile(authDbgPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
slog.Error("auth debug: cannot open log", "path", authDbgPath, "error", err)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
f.WriteString(line)
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
// Package webserver provides a server for habits tracking functionality through Telegram miniapps.
|
||||
// SSLs certificates are handled automatically via LetsEncrypt.
|
||||
package sync
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"crypto/tls"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/zakirullin/files.md/server/config"
|
||||
"github.com/zakirullin/files.md/server/fs"
|
||||
"github.com/zakirullin/files.md/server/habits"
|
||||
"github.com/zakirullin/files.md/server/journal"
|
||||
"github.com/zakirullin/files.md/server/userconfig"
|
||||
)
|
||||
|
||||
type gzipResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
gz *gzip.Writer
|
||||
}
|
||||
|
||||
func (w gzipResponseWriter) Write(b []byte) (int, error) { return w.gz.Write(b) }
|
||||
|
||||
// gzipMiddleware streams the response through gzip when the client advertises
|
||||
// gzip support. No-op otherwise.
|
||||
func gzipMiddleware(h http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
|
||||
h(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
w.Header().Del("Content-Length")
|
||||
gz := gzip.NewWriter(w)
|
||||
defer gz.Close()
|
||||
h(gzipResponseWriter{ResponseWriter: w, gz: gz}, r)
|
||||
}
|
||||
}
|
||||
|
||||
// Serve TODO release graceful shutdown etc
|
||||
// All directories paths are absolute.
|
||||
func Serve(apiHost, appHost, certDir, logFilename string) {
|
||||
// Logger is used for ssl/connection errors.
|
||||
// For regular errors we still use slog.
|
||||
serverLogger := newLogger(logFilename)
|
||||
|
||||
serverLogger.Printf("Resolved hosts: api_host=%q app_host=%q cert_dir=%q", apiHost, appHost, certDir)
|
||||
|
||||
// For local environment.
|
||||
// TODO make it more explicit
|
||||
if certDir == "" {
|
||||
srv := &http.Server{
|
||||
Addr: ":8080",
|
||||
Handler: router(serverLogger),
|
||||
}
|
||||
|
||||
serverLogger.Printf("Starting HTTP server on %s", srv.Addr)
|
||||
err := srv.ListenAndServe()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// This will also launch :80 http server that would pass ACME challenges or redirects to :443.
|
||||
autocert := certServer(serverLogger, certDir, apiHost, appHost)
|
||||
tlsConfig := &tls.Config{
|
||||
GetCertificate: autocert.GetCertificate,
|
||||
CurvePreferences: []tls.CurveID{tls.X25519, tls.CurveP256},
|
||||
}
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: ":443",
|
||||
TLSConfig: tlsConfig,
|
||||
IdleTimeout: 2 * time.Minute,
|
||||
ReadTimeout: 30 * time.Second, // Otherwise we get net::ERR_HTTP2_PROTOCOL_ERROR (RST_STREAM) errors on slow clients (I personally experienced it in South America on syncMedia upload)
|
||||
WriteTimeout: 2 * time.Minute, // For slow files like inbox.wasm.
|
||||
ErrorLog: serverLogger,
|
||||
}
|
||||
srv.Handler = router(serverLogger)
|
||||
|
||||
serverLogger.Printf("Starting HTTPS server on %s (api_host=%q app_host=%q cert_dir=%q)", srv.Addr, apiHost, appHost, certDir)
|
||||
err := srv.ListenAndServeTLS("", "") // Key and cert provided automatically by autocert
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func router(serverLogger *log.Logger) *http.ServeMux {
|
||||
r := http.NewServeMux()
|
||||
|
||||
// TODO add hashing or secrets
|
||||
// TODO before release habits_v2 => habits
|
||||
r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
// Log Range requests
|
||||
if rangeHeader := r.Header.Get("Range"); rangeHeader != "" {
|
||||
serverLogger.Printf("🔍 Range request: %s %s - Range: %s", r.Method, r.URL.Path, rangeHeader)
|
||||
serverLogger.Printf("📱 User-Agent: %s", r.Header.Get("User-Agent"))
|
||||
}
|
||||
|
||||
// Serving the PWA app
|
||||
if appHost := config.ServerCfg.AppHost(); appHost != "" && r.Host == appHost {
|
||||
http.FileServer(http.Dir("./web")).ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
// Sync API is on only when API_URL is not empty.
|
||||
if config.ServerCfg.APIHost() != "" {
|
||||
r.HandleFunc("/syncFilenames", corsMiddleware(panicMiddleware(tokenMiddleware(gzipMiddleware(SyncFilenames)))))
|
||||
r.HandleFunc("/syncFile", corsMiddleware(panicMiddleware(tokenMiddleware(gzipMiddleware(SyncFile)))))
|
||||
r.HandleFunc("/syncMediaFilenames", corsMiddleware(panicMiddleware(tokenMiddleware(gzipMiddleware(SyncMediaFilenames)))))
|
||||
r.HandleFunc("/syncMediaFile", corsMiddleware(panicMiddleware(tokenMiddleware(gzipMiddleware(SyncMediaFile)))))
|
||||
r.HandleFunc("/issuePermanentToken", corsMiddleware(panicMiddleware(IssueToken)))
|
||||
|
||||
// Deprecated due to cryptic names :) Will be removed soon.
|
||||
r.HandleFunc("/syncTexts", corsMiddleware(panicMiddleware(tokenMiddleware(gzipMiddleware(SyncFilenames)))))
|
||||
r.HandleFunc("/syncText", corsMiddleware(panicMiddleware(tokenMiddleware(gzipMiddleware(SyncFile)))))
|
||||
r.HandleFunc("/syncMedias", corsMiddleware(panicMiddleware(tokenMiddleware(gzipMiddleware(SyncMediaFilenames)))))
|
||||
r.HandleFunc("/syncMedia", corsMiddleware(panicMiddleware(tokenMiddleware(gzipMiddleware(SyncMediaFile)))))
|
||||
r.HandleFunc("/token", corsMiddleware(panicMiddleware(IssueToken)))
|
||||
}
|
||||
|
||||
// For now it is possible to see other user's habits, but is it a big deal?
|
||||
// TODO use X-Telegram-Init-Data header to verify requests
|
||||
r.HandleFunc("GET /habits_v2/{userID}", func(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := strconv.ParseInt(r.PathValue("userID"), 10, 64)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to parse userID for habits: %v", err)
|
||||
http.Error(w, "can't parse userID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
userFS, err := fs.NewUserFS(userID)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to init userFS: %v", err)
|
||||
http.Error(w, "can't init userFS", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
str, err := habits.Render(userID, userFS)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to render habits: %v", err)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_, err = w.Write(str)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to write habits response: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// TODO use X-Telegram-Init-Data header to verify requests
|
||||
r.HandleFunc("POST /habits_v2/{userID}/{habitName}/{yearDay}/{status}", func(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := strconv.ParseInt(r.PathValue("userID"), 10, 64)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to parse userID: %v", err)
|
||||
http.Error(w, "can't parse userID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
yearDay, err := strconv.ParseInt(r.PathValue("yearDay"), 10, 32)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to parse yearDay: %v", err)
|
||||
http.Error(w, "can't parse yearDay", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
status, err := strconv.ParseInt(r.PathValue("status"), 10, 32)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to parse status: %v", err)
|
||||
http.Error(w, "can't parse status", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
habitName := r.PathValue("habitName")
|
||||
|
||||
userFS, err := fs.NewUserFS(userID)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to init user fs: %v", err)
|
||||
http.Error(w, "can't init user fs", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
userHabits, err := habits.Habits(userFS, time.Now().Year())
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to read habits: %v", err)
|
||||
http.Error(w, "can't read habits", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := userHabits[habitName]; !ok {
|
||||
userHabits[habitName] = make(habits.Year)
|
||||
}
|
||||
userHabits[habitName][int(yearDay)] = int(status)
|
||||
err = habits.Write(userFS, time.Now().Year(), userHabits)
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to write habits: %v", err)
|
||||
http.Error(w, "can't write habits", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
emoji := habits.Emoji(userFS, habitName)
|
||||
if habitName == habits.MoodHabit {
|
||||
if int(status) < len(habits.MoodEmojis) {
|
||||
emoji = habits.MoodEmojis[status]
|
||||
}
|
||||
}
|
||||
|
||||
userConf := userconfig.NewConfig(userFS, userID, config.ServerCfg.ConfigFilename)
|
||||
err = journal.AddEmoji(userFS, emoji, userConf.Timezone())
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to write habit emoji to journal: %v", err)
|
||||
http.Error(w, "can't write habit emoji to journal", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
record := fmt.Sprintf("%s %s", emoji, habitName)
|
||||
err = journal.AddRecord(userFS, record, userConf.Timezone())
|
||||
if err != nil {
|
||||
serverLogger.Printf("failed to write habit to journal: %v", err)
|
||||
http.Error(w, "can't write habit to journal", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func newLogger(logFilename string) *log.Logger {
|
||||
logFile, err := os.OpenFile(logFilename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o666)
|
||||
if err != nil {
|
||||
log.Fatalf("Server: failed to open log file: %v", err)
|
||||
}
|
||||
|
||||
filteredWriter := &FilteredWriter{
|
||||
writer: logFile,
|
||||
ignorePatterns: []string{
|
||||
"TLS handshake error",
|
||||
},
|
||||
}
|
||||
|
||||
return log.New(filteredWriter, "Server Error: ", log.Ldate|log.Ltime|log.Lshortfile)
|
||||
}
|
||||
|
||||
type FilteredWriter struct {
|
||||
writer io.Writer
|
||||
ignorePatterns []string
|
||||
}
|
||||
|
||||
func (fw *FilteredWriter) Write(p []byte) (n int, err error) {
|
||||
message := string(p)
|
||||
for _, pattern := range fw.ignorePatterns {
|
||||
if strings.Contains(message, pattern) {
|
||||
return len(p), nil
|
||||
}
|
||||
}
|
||||
|
||||
return fw.writer.Write(p)
|
||||
}
|
||||
|
||||
func panicMiddleware(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
slog.Error("Handler panic",
|
||||
slog.String("method", r.Method),
|
||||
slog.String("path", r.URL.Path),
|
||||
slog.Any("panic", err),
|
||||
slog.String("stack", string(debug.Stack())),
|
||||
)
|
||||
|
||||
if w.Header().Get("Content-Type") == "" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
}
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(`{"error":"Internal server error"}`))
|
||||
}
|
||||
}()
|
||||
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func corsMiddleware(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == config.ServerCfg.AppURL {
|
||||
w.Header().Set("Access-Control-Allow-Origin", origin)
|
||||
}
|
||||
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization, X-CSRF-OneTimeToken, Version")
|
||||
w.Header().Set("Vary", "Origin")
|
||||
|
||||
if r.Method == "OPTIONS" {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string("0 \n000")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string(".")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string("1000000")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string(". ")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string("\x85")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string(" ..")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string(" jj")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string("000000\xf0\x9f\x00\x80\x80")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string("\xd0\xd9000\xf40\xea0000\x910\xb0\x87\xae00000\xd00\x8d0\xcf0\x8d0000\xaf0000000000\xb2\xd7\xe5\x8d000000\xc3\xcf0\xfd\x98")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string("0000000000000000000000000000000000000000000000000000000000000000000000000000ˌ00000000000000000000000")
|
||||
@@ -0,0 +1,2 @@
|
||||
go test fuzz v1
|
||||
string("\n0")
|
||||
@@ -0,0 +1,3 @@
|
||||
go test fuzz v1
|
||||
string(" ")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user