core/remote/watcher: Fetch resynced folders content (#2100)

When a folder has been unsynced via the differential sync API, it is
seen as deleted by the Desktop client and so are its children.

However, when the same folder is resynced, the Desktop client will
only see a change for the folder.

To make sure we'll resync all its content, we will fetch it and
process all found remote documents as additions.
The way we detect a folder for which we need to fetch the content is
if we detect the addition of a folder whose remote revision is greater
than 1 (i.e. it was not just created on the Cozy).

If a new folder is modified on the Cozy before the client has detected
its addition via the changes feed, we'll treat it as a resynced folder
and fetch its content while we already have it in the feed.
We expect this not to be an issue though. the remote watcher analysis
should detect the second occurrence as an up-to-date change since the
remote revision will be the same.

core/config.js

@@ -164,6 +164,12 @@ class Config {
     }
   }
 
+  // Flags are options that can be activated by the user via the config file.
+  // They can be used to activate incomplete features for example.
+  get flags() /*: { [string]: boolean } */ {
+    return _.get(this.fileConfig, 'flags', {})
+  }
+
   get version() /*: ?string */ {
     return _.get(this.fileConfig, 'creds.client.softwareVersion', '')
   }

core/merge.js

@@ -201,17 +201,10 @@ class Merge {
         doc._rev = file._rev
 
         // Keep other side metadata if we're updating the deleted side of file
-        if (
-          side === 'remote' &&
-          file.remote &&
-          (file.remote._deleted || file.remote.trashed)
-        ) {
+        if (side === 'remote' && file.remote && file.remote.trashed) {
           doc.local = file.local
           metadata.markSide(side, doc, file)
-        } else if (
-          side === 'local' &&
-          (!file.remote || (!file.remote._deleted && !file.remote.trashed))
-        ) {
+        } else if (side === 'local' && (!file.remote || !file.remote.trashed)) {
           doc.remote = file.remote
           metadata.markSide(side, doc, file)
         } else {

core/metadata.js

@@ -97,7 +97,7 @@ export type MetadataLocalInfo = {
   updated_at?: string,
 }
 
-export type MetadataRemoteFile = { ...RemoteFile, path: string }
+export type MetadataRemoteFile = {| ...RemoteFile, path: string |}
 export type MetadataRemoteDir = RemoteDir
 export type MetadataRemoteInfo = MetadataRemoteFile|MetadataRemoteDir
 
@@ -122,7 +122,7 @@ export type Metadata = {
   path: string,
   updated_at: string,
   local: MetadataLocalInfo,
-  remote: MetadataRemoteDir|MetadataRemoteFile,
+  remote: MetadataRemoteInfo,
   tags: string[],
   sides: MetadataSidesInfo,
 
@@ -468,8 +468,7 @@ function ensureValidChecksum(doc /*: Metadata */) {
 // Extract the revision number, or 0 it not found
 function extractRevNumber(doc /*: { _rev: string } */) {
   try {
-    // $FlowFixMe
-    let rev = doc._rev.split('-')[0]
+    const rev = doc._rev.split('-')[0]
     return Number(rev)
   } catch (error) {
     return 0

core/remote/change.js

@@ -79,6 +79,11 @@ export type RemoteDirTrashing = {
   doc: Metadata,
   was: SavedMetadata
 }
+export type RemoteDirUpdate = {
+  sideName: 'remote',
+  type: 'DirUpdate',
+  doc: Metadata
+}
 export type RemoteIgnoredChange = {
   sideName: 'remote',
   type: 'IgnoredChange',
@@ -115,6 +120,7 @@ export type RemoteChange =
   | RemoteDirMove
   | RemoteDirRestoration
   | RemoteDirTrashing
+  | RemoteDirUpdate
   | RemoteFileAddition
   | RemoteFileDeletion
   | RemoteFileMove
@@ -212,7 +218,7 @@ function upToDate(
 
 function updated(
   doc /*: Metadata */
-) /*: RemoteFileUpdate | RemoteDirAddition */ {
+) /*: RemoteFileUpdate | RemoteDirUpdate */ {
   if (metadata.isFile(doc)) {
     return {
       sideName,
@@ -222,7 +228,7 @@ function updated(
   } else {
     return {
       sideName,
-      type: 'DirAddition',
+      type: 'DirUpdate',
       doc
     }
   }

core/remote/constants.js

@@ -7,6 +7,7 @@
 /*::
 export type FILE_TYPE = 'file'
 export type DIR_TYPE = 'directory'
+export type FILES_DOCTYPE = 'io.cozy.files'
 */
 
 const DEFAULT_HEARTBEAT = 1000 * 60 // 1 minute

core/remote/cozy.js

@@ -13,7 +13,8 @@ const { FILES_DOCTYPE, FILE_TYPE, DIR_TYPE } = require('./constants')
 const { DirectoryNotFound } = require('./errors')
 const {
   dropSpecialDocs,
-  jsonApiToRemoteDoc,
+  remoteJsonToRemoteDoc,
+  jsonApiToRemoteJsonDoc,
   keepFiles,
   parentDirIds
 } = require('./document')
@@ -22,7 +23,15 @@ const logger = require('../utils/logger')
 /*::
 import type { Config } from '../config'
 import type { Readable } from 'stream'
-import type { JsonApiDoc, RemoteDoc, RemoteFile, RemoteDir, RemoteDeletion } from './document'
+import type {
+  RemoteJsonDoc,
+  RemoteJsonFile,
+  RemoteJsonDir,
+  RemoteDoc,
+  RemoteFile,
+  RemoteDir,
+  RemoteDeletion,
+} from './document'
 import type { MetadataRemoteInfo, MetadataRemoteFile, MetadataRemoteDir } from '../metadata'
 
 export type Warning = {
@@ -38,6 +47,12 @@ export type Reference = {
   id: string,
   type: string
 }
+
+type ChangesFeedResponse = {|
+  pending: number,
+  last_seq: string,
+  results: Array<{ doc: RemoteDoc|RemoteDeletion }>
+|}
 */
 
 const log = logger({
@@ -169,7 +184,7 @@ class RemoteCozy {
                  executable: boolean|} */
   ) /*: Promise<MetadataRemoteFile> */ {
     return this._withDomainErrors(options, async () => {
-      const file = await this.client.files.create(data, {
+      const file /* RemoteJsonFile*/ = await this.client.files.create(data, {
         ...options,
         noSanitize: true
       })
@@ -183,10 +198,12 @@ class RemoteCozy {
                  createdAt: string,
                  updatedAt: string|} */
   ) /*: Promise<MetadataRemoteDir> */ {
-    const folder = await this.client.files.createDirectory({
-      ...options,
-      noSanitize: true
-    })
+    const folder /*: RemoteJsonDir */ = await this.client.files.createDirectory(
+      {
+        ...options,
+        noSanitize: true
+      }
+    )
     return this.toRemoteDoc(folder)
   }
 
@@ -201,10 +218,14 @@ class RemoteCozy {
                  ifMatch: string|} */
   ) /*: Promise<MetadataRemoteFile> */ {
     return this._withDomainErrors(options, async () => {
-      const updated = await this.client.files.updateById(id, data, {
-        ...options,
-        noSanitize: true
-      })
+      const updated /*: RemoteJsonFile */ = await this.client.files.updateById(
+        id,
+        data,
+        {
+          ...options,
+          noSanitize: true
+        }
+      )
       return this.toRemoteDoc(updated)
     })
   }
@@ -245,37 +266,46 @@ class RemoteCozy {
     const { last_seq, results } = await getChangesFeed(since, this.client)
 
     // The stack docs: dirs, files (without a path), deletions
-    const rawDocs /*: RemoteDoc[] */ = dropSpecialDocs(results.map(r => r.doc))
+    const remoteDocs /*: Array<RemoteDoc|RemoteDeletion> */ = dropSpecialDocs(
+      results.map(r => r.doc)
+    )
+    const docs = await this.completeRemoteDocs(remoteDocs)
+
+    return { last_seq, docs }
+  }
+
+  async completeRemoteDocs(
+    rawDocs /*: Array<RemoteDoc|RemoteDeletion> */
+  ) /*: Promise<Array<MetadataRemoteInfo|RemoteDeletion>> */ {
+    // The final docs with their paths (except for deletions)
+    const remoteDocs /*: Array<MetadataRemoteInfo|RemoteDeletion> */ = []
 
     // The parent dirs for each file, indexed by id
     const fileParentsById = await this.client.data.findMany(
       FILES_DOCTYPE,
       parentDirIds(keepFiles(rawDocs))
     )
 
-    // The final docs with their paths (except for deletions)
-    const remoteDocs /*: Array<MetadataRemoteInfo|RemoteDeletion> */ = []
-
-    for (const remoteDoc of rawDocs) {
-      if (remoteDoc.type === FILE_TYPE) {
-        // File docs returned by the cozy-stack don't have a path
-        const parent = fileParentsById[remoteDoc.dir_id]
-
+    for (const rawDoc of rawDocs) {
+      if (rawDoc._deleted) {
+        remoteDocs.push(rawDoc)
+      } else if (rawDoc.type === FILE_TYPE) {
+        const parent = fileParentsById[rawDoc.dir_id]
         if (parent.error || parent.doc == null || parent.doc.path == null) {
           log.error(
-            { err: parent.error, remoteDoc, parent, sentry: true },
+            { err: parent.error, rawDoc, parent, sentry: true },
             'Could not compute doc path from parent'
           )
           continue
         } else {
-          remoteDocs.push(this._withPath(remoteDoc, parent.doc))
+          remoteDocs.push(this._withPath(rawDoc, parent.doc))
         }
       } else {
-        remoteDocs.push(remoteDoc)
+        remoteDocs.push(rawDoc)
       }
     }
 
-    return { last_seq, docs: remoteDocs }
+    return remoteDocs
   }
 
   async find(id /*: string */) /*: Promise<MetadataRemoteInfo> */ {
@@ -341,6 +371,44 @@ class RemoteCozy {
     return results[0]
   }
 
+  async getDirectoryContent(
+    dir /*: RemoteDir */,
+    { client } /*: { client: ?CozyClient } */ = {}
+  ) /*: Promise<$ReadOnlyArray<MetadataRemoteInfo>> */ {
+    client = client || (await this.newClient())
+
+    let dirContent = []
+    let resp /*: { next: boolean, bookmark?: string, data: Object[] } */ = {
+      next: true,
+      data: []
+    }
+    while (resp && resp.next) {
+      const queryDef = client
+        .find(FILES_DOCTYPE)
+        .where({
+          dir_id: dir._id
+        })
+        .indexFields(['name'])
+        .sortBy([{ name: 'asc' }])
+        .limitBy(10000)
+        .offsetBookmark(resp.bookmark)
+      resp = await client.query(queryDef)
+      for (const j of resp.data) {
+        const remoteJson = jsonApiToRemoteJsonDoc(j)
+        if (remoteJson._deleted) continue
+
+        const remoteDoc = await this.toRemoteDoc(remoteJson, dir)
+        dirContent.push(remoteDoc)
+        if (remoteDoc.type === DIR_TYPE) {
+          // Fetch subdir content
+          dirContent.push(this.getDirectoryContent(remoteDoc, { client }))
+        }
+      }
+    }
+    // $FlowFixMe Array.prototype.flat is available in NodeJS v12
+    return (await Promise.all(dirContent)).flat()
+  }
+
   async isEmpty(id /*: string */) /*: Promise<boolean> */ {
     const dir = await this.client.files.statById(id)
     if (dir.attributes.type !== 'directory') {
@@ -357,13 +425,16 @@ class RemoteCozy {
     return resp.body
   }
 
-  async toRemoteDoc /*:: <T: JsonApiDoc> */(doc /*: T */) /*: Promise<*> */ {
-    const remoteDoc /*: RemoteDoc */ = jsonApiToRemoteDoc(doc)
+  async toRemoteDoc(
+    doc /*: RemoteJsonDoc */,
+    parentDir /*: ?RemoteDir */
+  ) /*: Promise<*> */ {
+    const remoteDoc = remoteJsonToRemoteDoc(doc)
     if (remoteDoc.type === FILE_TYPE) {
-      const parentDir /*: RemoteDir */ = await this.findDir(remoteDoc.dir_id)
+      parentDir = parentDir || (await this.findDir(remoteDoc.dir_id))
       return this._withPath(remoteDoc, parentDir)
     }
-    return (remoteDoc /*: MetadataRemoteDir */)
+    return remoteDoc
   }
 
   /** Set the path of a remote file doc. */
@@ -445,7 +516,7 @@ class RemoteCozy {
 async function getChangesFeed(
   since /*: string */,
   client /*: OldCozyClient */
-) /*: Promise<{ pending: number, last_seq: string, results: Array<{ doc: RemoteDoc }> }> */ {
+) /*: Promise<ChangesFeedResponse> */ {
   const response = await client.data.changesFeed(FILES_DOCTYPE, {
     since,
     include_docs: true,