Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Ensure bookmark files are correctly downloaded before deleting current ones #683

Merged
merged 37 commits into from Aug 20, 2023
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
7aa37d2
generate ebook get dstPath
Monirzadeh Jul 28, 2023
91937b3
Archive and ebook can recover if download faild
Monirzadeh Jul 28, 2023
c12e3f2
recover thumb if download faild
Monirzadeh Aug 5, 2023
379a3ba
thumb image create just if image processing is sucssesful
Monirzadeh Aug 5, 2023
42afd23
create epub in tmp if it sucssesful copy to destination
Monirzadeh Aug 5, 2023
624a746
archive file create in tmp if it successful move to destination
Monirzadeh Aug 5, 2023
2ec8e5e
move to destination as function
Monirzadeh Aug 5, 2023
44b9a13
update ebook download api and remove .epub from file name
Monirzadeh Aug 5, 2023
622ff9d
report faild item to user
Monirzadeh Aug 5, 2023
bf31c3b
not show dialog if error not happen
Monirzadeh Aug 5, 2023
658c80f
update thumbnail based on last status of bookmark fix #524
Monirzadeh Aug 5, 2023
87b6861
better warning massage
Monirzadeh Aug 6, 2023
c54da26
tmpFile without .epub
Monirzadeh Aug 6, 2023
0a752e1
MoveToDestination change to MoveFileToDestination
Monirzadeh Aug 6, 2023
89c04d6
return .epub
Monirzadeh Aug 6, 2023
2b489ea
log if downloadBookImage return error
Monirzadeh Aug 6, 2023
3588119
fix bug remove imgPath just if download last image be unsuccessful
Monirzadeh Aug 6, 2023
81703cb
update old unit test
Monirzadeh Aug 6, 2023
e56ef8f
add processing.go unit test
Monirzadeh Aug 6, 2023
947eefa
small massage for report failded item to the user
Monirzadeh Aug 6, 2023
757599f
add some more unit test and samplefile
Monirzadeh Aug 7, 2023
d33384a
use sample image in unit test
Monirzadeh Aug 7, 2023
d1a6412
use local sample file and unit test not need internet connection anymore
Monirzadeh Aug 7, 2023
a44caad
update error to user and log that too
Monirzadeh Aug 12, 2023
612c439
add more comment
Monirzadeh Aug 12, 2023
9a42103
update comment
Monirzadeh Aug 12, 2023
55bb071
change variable name parentDir to dstDir
Monirzadeh Aug 14, 2023
3da6206
more simpler error handling
Monirzadeh Aug 14, 2023
1630092
remove unneeded defer
Monirzadeh Aug 14, 2023
afba6ff
remvoe unneeded epubWriter.Close()
Monirzadeh Aug 14, 2023
17f984d
more readable unit test in processing
Monirzadeh Aug 14, 2023
bf95321
more readable unit test for ebooks
Monirzadeh Aug 14, 2023
7f26e3f
delete all defer os.RemoveAll from unit tests
Monirzadeh Aug 14, 2023
9341e4d
Merge branch 'master' into cache-recovery
Monirzadeh Aug 14, 2023
cdf6fab
Better comment
Monirzadeh Aug 20, 2023
587340b
Better Error output
Monirzadeh Aug 20, 2023
3003099
fix err.String() method
Monirzadeh Aug 20, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
49 changes: 29 additions & 20 deletions internal/core/ebook.go
Expand Up @@ -16,7 +16,7 @@ import (
"github.com/pkg/errors"
)

func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) {
func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err error) {
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
// variable for store generated html code
var html string

Expand All @@ -40,34 +40,22 @@ func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) {
if _, err := os.Stat(archivePath); err == nil {
book.HasArchive = true
}
ebookfile := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID))
// if epub exist finish prosess else continue
if _, err := os.Stat(ebookfile); err == nil {
book.HasEbook = true
return book, nil
}

contentType := req.ContentType
if strings.Contains(contentType, "application/pdf") {
return book, errors.New("can't create ebook for pdf")
}

ebookDir := fp.Join(req.DataDir, "ebook")
// check if directory not exsist create that
if _, err := os.Stat(ebookDir); os.IsNotExist(err) {
err := os.MkdirAll(ebookDir, model.DataDirPerm)
if err != nil {
return book, errors.Wrap(err, "can't create ebook directory")
}
}
// create epub file
epubFile, err := os.Create(ebookfile)
// create temporary epub file
tmpFile, err := os.CreateTemp("", "ebook*.epub")
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return book, errors.Wrap(err, "can't create ebook")
return book, errors.Wrap(err, "can't create temporary EPUB file")
}
defer epubFile.Close()
defer tmpFile.Close()
defer os.Remove(tmpFile.Name())
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved

// Create zip archive
epubWriter := zip.NewWriter(epubFile)
epubWriter := zip.NewWriter(tmpFile)
defer epubWriter.Close()

// Create the mimetype file
Expand Down Expand Up @@ -223,6 +211,27 @@ img {
if err != nil {
return book, errors.Wrap(err, "can't write into content.html")
}
// close epub and tmpFile
err = epubWriter.Close()
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return book, errors.Wrap(err, "failed to close EPUB writer")
}
err = tmpFile.Close()
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return book, errors.Wrap(err, "failed to close temporary EPUB file")
}
// open temporary file again
tmpFile, err = os.Open(tmpFile.Name())
fmartingr marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return book, errors.Wrap(err, "can't open temporary EPUB file")
}
defer tmpFile.Close()

err = MoveToDestination(dstPath, tmpFile)
if err != nil {
return book, errors.Wrap(err, "failed move ebook to destination")
}

book.HasEbook = true
return book, nil
}
Expand Down
73 changes: 55 additions & 18 deletions internal/core/processing.go
Expand Up @@ -72,7 +72,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,

nurl, err := url.Parse(book.URL)
if err != nil {
return book, true, fmt.Errorf("Failed to parse url: %v", err)
return book, true, fmt.Errorf("failed to parse url: %v", err)
}

article, err := readability.FromReader(readabilityInput, nurl)
Expand Down Expand Up @@ -128,13 +128,12 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,

// If needed, create ebook as well
if book.CreateEbook {
ebookPath := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID))
os.Remove(ebookPath)
ebookPath := fp.Join(req.DataDir, "ebook", strID)

if strings.Contains(contentType, "application/pdf") {
return book, false, errors.Wrap(err, "can't create ebook from pdf")
} else {
_, err = GenerateEbook(req)
_, err = GenerateEbook(req, ebookPath)
if err != nil {
return book, true, errors.Wrap(err, "failed to create ebook")
}
Expand All @@ -144,8 +143,11 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,

// If needed, create offline archive as well
if book.CreateArchive {
archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))
os.Remove(archivePath)
tmpFile, err := os.CreateTemp("", "archive")
if err != nil {
return book, false, fmt.Errorf("failed to create temp archive: %v", err)
}
defer os.Remove(tmpFile.Name())

archivalRequest := warc.ArchivalRequest{
URL: book.URL,
Expand All @@ -155,11 +157,20 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
LogEnabled: req.LogArchival,
}

err = warc.NewArchive(archivalRequest, archivePath)
err = warc.NewArchive(archivalRequest, tmpFile.Name())
if err != nil {
defer os.Remove(tmpFile.Name())
return book, false, fmt.Errorf("failed to create archive: %v", err)
}

// Prepare destination file.
dstPath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))

err = MoveToDestination(dstPath, tmpFile)
if err != nil {
return book, false, fmt.Errorf("failed move archive to destination `: %v", err)
}

book.HasArchive = true
}

Expand All @@ -185,17 +196,12 @@ func downloadBookImage(url, dstPath string) error {
}

// At this point, the download has finished successfully.
// Prepare destination file.
err = os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm)
// Create tmpFile
tmpFile, err := os.CreateTemp("", "image")
if err != nil {
return fmt.Errorf("failed to create image dir: %v", err)
return fmt.Errorf("failed to create temporary image file: %v", err)
}

dstFile, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("failed to create image file: %v", err)
}
defer dstFile.Close()
defer os.Remove(tmpFile.Name())

// Parse image and process it.
// If image is smaller than 600x400 or its ratio is less than 4:3, resize.
Expand All @@ -211,7 +217,7 @@ func downloadBookImage(url, dstPath string) error {
imgRatio := float64(imgWidth) / float64(imgHeight)

if imgWidth >= 600 && imgHeight >= 400 && imgRatio > 1.3 {
err = jpeg.Encode(dstFile, img, nil)
err = jpeg.Encode(tmpFile, img, nil)
} else {
// Create background
bg := image.NewNRGBA(imgRect)
Expand All @@ -236,12 +242,43 @@ func downloadBookImage(url, dstPath string) error {
draw.Draw(bg, bgRect, fg, fgPosition, draw.Over)

// Save to file
err = jpeg.Encode(dstFile, bg, nil)
err = jpeg.Encode(tmpFile, bg, nil)
}

if err != nil {
return fmt.Errorf("failed to save image %s: %v", url, err)
}

err = MoveToDestination(dstPath, tmpFile)
if err != nil {
return err
}

return nil
}

func MoveToDestination(dstPath string, tmpFile *os.File) error {
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
// Prepare destination file.
err := os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm)
if err != nil {
return fmt.Errorf("failed to create destination dir: %v", err)
}

dstFile, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("failed to create destination file: %v", err)
}
defer dstFile.Close()
// Copy temporary file to destination
_, err = tmpFile.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("failed to rewind temporary file: %v", err)
}

_, err = io.Copy(dstFile, tmpFile)
if err != nil {
return fmt.Errorf("failed to copy file to the destination")
}

return nil
}
38 changes: 29 additions & 9 deletions internal/view/assets/js/page/home.js
Expand Up @@ -702,15 +702,35 @@ export default {
this.editMode = false;
this.dialog.loading = false;
this.dialog.visible = false;

json.forEach(book => {
var item = items.find(el => el.id === book.id);
this.bookmarks.splice(item.index, 1, book);
});
}).catch(err => {
this.selection = [];
this.editMode = false;
this.dialog.loading = false;

Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
let faildUpdateArchives = [];
let faildCreateEbook = [];
json.forEach(book => {
var item = items.find(el => el.id === book.id);
this.bookmarks.splice(item.index, 1, book);

if (data.createArchive && !book.hasArchive){
faildUpdateArchives.push(book.id);
}
if (data.createEbook && !book.hasEbook){
faildCreateEbook.push(book.id);
}
}),

this.showDialog({
title: `Update Archive Error`,
content: `Bookmarks Update Archive Faild : ${faildUpdateArchives.join(", ")}
Bookmarks Create Ebook Faild: ${faildCreateEbook.join(",")}
We recovered the last available version.`,
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
mainText: "OK",
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
mainClick: () => {
this.dialog.visible = false;
},
})
}).catch(err => {
this.selection = [];
this.editMode = false;
this.dialog.loading = false;

this.getErrorMessage(err).then(msg => {
this.showErrorDialog(msg);
Expand Down
34 changes: 27 additions & 7 deletions internal/webserver/handler-api.go
Expand Up @@ -110,7 +110,7 @@ func (h *Handler) ApiGetBookmarks(w http.ResponseWriter, r *http.Request, ps htt
strID := strconv.Itoa(bookmarks[i].ID)
imgPath := fp.Join(h.DataDir, "thumb", strID)
archivePath := fp.Join(h.DataDir, "archive", strID)
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
ebookPath := fp.Join(h.DataDir, "ebook", strID)
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved

if fileExists(imgPath) {
bookmarks[i].ImageURL = path.Join(h.RootPath, "bookmark", strID, "thumb")
Expand Down Expand Up @@ -285,7 +285,7 @@ func (h *Handler) ApiDeleteBookmark(w http.ResponseWriter, r *http.Request, ps h
strID := strconv.Itoa(id)
imgPath := fp.Join(h.DataDir, "thumb", strID)
archivePath := fp.Join(h.DataDir, "archive", strID)
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
ebookPath := fp.Join(h.DataDir, "ebook", strID)
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved

os.Remove(imgPath)
os.Remove(archivePath)
Expand Down Expand Up @@ -434,12 +434,32 @@ func (h *Handler) ApiDownloadEbook(w http.ResponseWriter, r *http.Request, ps ht
ContentType: contentType,
}

book, err = core.GenerateEbook(request)
content.Close()
// if file exist book return avilable file
strID := strconv.Itoa(book.ID)
ebookPath := fp.Join(request.DataDir, "ebook", strID)
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
_, err = os.Stat(ebookPath)
if err == nil {
// file already exists, return the existing file
imagePath := fp.Join(request.DataDir, "thumb", fmt.Sprintf("%d", book.ID))
archivePath := fp.Join(request.DataDir, "archive", fmt.Sprintf("%d", book.ID))

if _, err := os.Stat(imagePath); err == nil {
book.ImageURL = fp.Join("/", "bookmark", strID, "thumb")
}

if err != nil {
chProblem <- book.ID
return
if _, err := os.Stat(archivePath); err == nil {
book.HasArchive = true
}
book.HasEbook = true
} else {
// generate ebook file
book, err = core.GenerateEbook(request, ebookPath)
content.Close()

if err != nil {
chProblem <- book.ID
return
}
}

// Update list of bookmarks
Expand Down
6 changes: 3 additions & 3 deletions internal/webserver/handler-ui.go
Expand Up @@ -48,8 +48,8 @@ func (h *Handler) ServeBookmarkContent(w http.ResponseWriter, r *http.Request, p
}
}

// Check if it has archive.
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
// Check if it has ebook.
ebookPath := fp.Join(h.DataDir, "ebook", strID)
if fileExists(ebookPath) {
bookmark.HasEbook = true
}
Expand Down Expand Up @@ -320,7 +320,7 @@ func (h *Handler) ServeBookmarkEbook(w http.ResponseWriter, r *http.Request, ps
}

// Check if it has ebook.
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
ebookPath := fp.Join(h.DataDir, "ebook", strID)
Monirzadeh marked this conversation as resolved.
Show resolved Hide resolved
if !fileExists(ebookPath) {
http.Error(w, "ebook not found", http.StatusNotFound)
return
Expand Down