diff --git a/app/controllers/documents_controller.rb b/app/controllers/documents_controller.rb index f484adfa..a4f7b4cc 100644 --- a/app/controllers/documents_controller.rb +++ b/app/controllers/documents_controller.rb @@ -4,8 +4,6 @@ class DocumentsController < ApplicationController # todo Uh, this is a hack. The CSRF token on document editor model to add entities is being rejected... for whatever reason. skip_before_action :verify_authenticity_token, only: [:link_entity] - skip_before_action :cache_most_used_page_information, only: [:update] - before_action :set_document, only: [:show, :analysis, :plaintext, :queue_analysis, :edit, :destroy] before_action :set_sidenav_expansion, except: [:plaintext] before_action :set_navbar_color, except: [:plaintext] diff --git a/app/controllers/export_controller.rb b/app/controllers/export_controller.rb index b9a20063..ac3c2f5f 100644 --- a/app/controllers/export_controller.rb +++ b/app/controllers/export_controller.rb @@ -2,6 +2,9 @@ class ExportController < ApplicationController before_action :authenticate_user! before_action :whitelist_pluralized_model, only: [:csv] + skip_before_action :cache_most_used_page_information, only: [:outline, :notebook_json] + skip_before_action :cache_forums_unread_counts, only: [:outline, :notebook_json] + def index @sidenav_expansion = 'my account' end @@ -11,11 +14,17 @@ class ExportController < ApplicationController end def outline - send_data content_to_outline, filename: "notebook-#{Date.today}.txt" + export = ExportService.text_outline_export(current_user.universes.pluck(:id)) + send_data export, filename: "notebook-#{Date.today}.txt" + end + + def markdown + export = ExportService.text_markdown_export(current_user.universes.pluck(:id)) + send_data export, filename: "notebook-#{Date.today}.md" end def notebook_json - json_dump = current_user.content.except('Document').except('Timeline').map { |category, content| {"#{category}": fill_relations(category.constantize, content)} }.to_json + json_dump = ExportService.json_export(current_user.universes.pluck(:id)) send_data json_dump, filename: "notebook-#{Date.today}.json" end diff --git a/app/jobs/export_job.rb b/app/jobs/export_job.rb new file mode 100644 index 00000000..0333ae1c --- /dev/null +++ b/app/jobs/export_job.rb @@ -0,0 +1,14 @@ +class ExportJob < ApplicationJob + queue_as :export + + def perform(*args) + universe_ids = args.shift + format = args.shift + + # design: left side 1/2 - list universes with toggle/checkboxes to include + # right side 1/2 - radios for format, checkboxes for options after + # (include hidden? include IDs? include blank fields?) + + + end +end diff --git a/app/models/concerns/has_page_references.rb b/app/models/concerns/has_page_references.rb index 8be704e4..cfc70595 100644 --- a/app/models/concerns/has_page_references.rb +++ b/app/models/concerns/has_page_references.rb @@ -6,8 +6,9 @@ module HasPageReferences included do # Pages that reference this one has_many :incoming_page_references, - as: :referenced_page, - class_name: PageReference.name + as: :referenced_page, + class_name: PageReference.name, + dependent: :destroy Rails.application.config.content_type_names[:all].each do |page_type| has_many "referencing_#{page_type.downcase}_pages".to_sym, through: :incoming_page_references, @@ -26,7 +27,8 @@ module HasPageReferences # Pages referenced by this one has_many :outgoing_page_references, as: :referencing_page, - class_name: PageReference.name + class_name: PageReference.name, + dependent: :destroy Rails.application.config.content_type_names[:all].each do |page_type| has_many "referenced_#{page_type.downcase}_pages".to_sym, through: :outgoing_page_references, diff --git a/app/models/page_data/attribute_field.rb b/app/models/page_data/attribute_field.rb index 5b4c34dc..4724503a 100644 --- a/app/models/page_data/attribute_field.rb +++ b/app/models/page_data/attribute_field.rb @@ -7,6 +7,8 @@ class AttributeField < ApplicationRecord belongs_to :attribute_category has_many :attribute_values, class_name: 'Attribute', dependent: :destroy + has_many :page_references, dependent: :destroy + validates_presence_of :user_id acts_as_list scope: [:user_id, :attribute_category_id] diff --git a/app/services/export_service.rb b/app/services/export_service.rb new file mode 100644 index 00000000..f9b16ebd --- /dev/null +++ b/app/services/export_service.rb @@ -0,0 +1,323 @@ +class ExportService < Service + INCLUDE_EMPTY_FIELD_LABELS = false + + def self.text_outline_export(universe_ids) + page_types = Rails.application.config.content_types[:all] + temporary_user_id_reference = Universe.find_by(id: universe_ids).user_id + + export_text = StringIO.new + + categories = fetch_categories(user_id: temporary_user_id_reference) + fields = fetch_fields(category_ids: categories.map(&:id)) + values = fetch_values(field_ids: fields.map(&:id)) + + page_types.each do |page_type| + # Get all pages in the given universe(s) + pages = (page_type.name == 'Universe') ? page_type.where(id: universe_ids) + : page_type.where(universe_id: universe_ids) + if pages.any? + export_text << "# #{page_type.name.pluralize}\n" + + categories_for_this_page_type = categories.select { |c| c.entity_type == page_type.name.downcase } + fields_for_this_page_type = fields.select { |f| categories_for_this_page_type.map(&:id).include? f.attribute_category_id } + + pages.each do |page| + export_text << "* Name: #{page.name}\n" + export_text << " ID: #{page.id}\n" + + values_for_this_page = values.select { |v| v.entity_type == page_type.name && v.entity_id == page.id } + + categories_for_this_page_type.each do |category| + # Delay printing our category header until we've found a value to print, so we don't print + # empty headers and blank sections. + printed_category_header = false + + fields_for_this_category = fields_for_this_page_type.select { |f| f.attribute_category_id == category.id } + values_for_these_fields = values_for_this_page.select { |v| fields_for_this_category.map(&:id).include? v.attribute_field_id } + + fields_for_this_category.each do |field| + value_for_this_field = values_for_these_fields.detect do |value| + value.attribute_field_id == field.id && + value.entity_type == page_type.name && + value.entity_id == page.id + end + + # If we have a field to print but haven't printed the header yet, print it + if value_for_this_field.try(:value).try(:present?) && !printed_category_header + export_text << " #{category.label}\n" + printed_category_header = true + end + + case field.field_type + when 'text_area', 'textarea' + # For answers to text fields, we can just output their value directly + if value_for_this_field.present? || INCLUDE_EMPTY_FIELD_LABELS + export_text << " #{field.label}: #{value_for_this_field.try :value}\n" + end + + when 'link' + # For link fields, we have link codes ([[Character-1]]) and need to translate + # them into the linked page's name in order for them to actually be useful + if value_for_this_field.present? + json_list = JSON.parse(value_for_this_field.value) + formatted_names = json_list.map do |link_code| + query_link_code_with_cache(*link_code.split('-')).name + end + + export_text << " #{field.label}: #{formatted_names.to_sentence}\n" + else + if INCLUDE_EMPTY_FIELD_LABELS + export_text << " #{field.label}:\n" + end + end + end + + end + end + + export_text << "\n" # spacer between pages + end + + export_text << "\n" # spacer between sections + end + end + + return export_text.string + end + + def self.text_markdown_export(universe_ids) + page_types = Rails.application.config.content_types[:all] + temporary_user_id_reference = Universe.find_by(id: universe_ids).user_id + + export_text = StringIO.new + + categories = fetch_categories(user_id: temporary_user_id_reference) + fields = fetch_fields(category_ids: categories.map(&:id)) + values = fetch_values(field_ids: fields.map(&:id)) + + page_types.each do |page_type| + # Get all pages in the given universe(s) + pages = (page_type.name == 'Universe') ? page_type.where(id: universe_ids) + : page_type.where(universe_id: universe_ids) + if pages.any? + export_text << "# #{page_type.name.pluralize}\n" + + categories_for_this_page_type = categories.select { |c| c.entity_type == page_type.name.downcase } + fields_for_this_page_type = fields.select { |f| categories_for_this_page_type.map(&:id).include? f.attribute_category_id } + + pages.each do |page| + export_text << "## #{page.name}\n" + export_text << "ID: #{page.id}\n" + + values_for_this_page = values.select { |v| v.entity_type == page_type.name && v.entity_id == page.id } + + categories_for_this_page_type.each do |category| + # Delay printing our category header until we've found a value to print, so we don't print + # empty headers and blank sections. + printed_category_header = false + + fields_for_this_category = fields_for_this_page_type.select { |f| f.attribute_category_id == category.id } + values_for_these_fields = values_for_this_page.select { |v| fields_for_this_category.map(&:id).include? v.attribute_field_id } + + fields_for_this_category.each do |field| + value_for_this_field = values_for_these_fields.detect do |value| + value.attribute_field_id == field.id && + value.entity_type == page_type.name && + value.entity_id == page.id + end + + # If we have a field to print but haven't printed the header yet, print it + if value_for_this_field.try(:value).try(:present?) && !printed_category_header + export_text << "### #{category.label}\n" + printed_category_header = true + end + + case field.field_type + when 'text_area', 'textarea' + # For answers to text fields, we can just output their value directly + if value_for_this_field.try(:value).present? || INCLUDE_EMPTY_FIELD_LABELS + export_text << "#### #{field.label}\n#{value_for_this_field.try :value}\n" + end + + when 'link' + # For link fields, we have link codes ([[Character-1]]) and need to translate + # them into the linked page's name in order for them to actually be useful + if value_for_this_field.present? + json_list = JSON.parse(value_for_this_field.value) + formatted_names = json_list.map do |link_code| + link_type, link_id = link_code.split('-') + page_ref = query_link_code_with_cache(link_type, link_id) + + "[#{page_ref.name}](https://www.notebook.ai/plan/#{link_type.downcase.pluralize}/#{link_id})" + end + + export_text << "#### #{field.label}\n#{formatted_names.to_sentence}\n" + else + if INCLUDE_EMPTY_FIELD_LABELS + export_text << "#### #{field.label}\n" + end + end + end + + end + end + + export_text << "\n" # spacer between pages + end + + export_text << "\n" # spacer between sections + end + end + + return export_text.string + end + + def self.csv_export(universe_ids, separator=',') + # todo: figure out zip + end + + def self.json_export(universe_ids) + page_types = Rails.application.config.content_types[:all] + temporary_user_id_reference = Universe.find_by(id: universe_ids).user_id + + export_object = {} + + categories = fetch_categories(user_id: temporary_user_id_reference) + fields = fetch_fields(category_ids: categories.map(&:id)) + values = fetch_values(field_ids: fields.map(&:id)) + + page_types.each do |page_type| + # Get all pages in the given universe(s) + pages = (page_type.name == 'Universe') ? page_type.where(id: universe_ids) + : page_type.where(universe_id: universe_ids) + if pages.any? + export_object[page_type.name] = [] + + categories_for_this_page_type = categories.select { |c| c.entity_type == page_type.name.downcase } + fields_for_this_page_type = fields.select { |f| categories_for_this_page_type.map(&:id).include? f.attribute_category_id } + + page_object = {} + + pages.each do |page| + page_object['name'] = page.name + page_object['id'] = page.id + + values_for_this_page = values.select { |v| v.entity_type == page_type.name && v.entity_id == page.id } + + categories_for_this_page_type.each do |category| + page_object[category.label] ||= {} + + fields_for_this_category = fields_for_this_page_type.select { |f| f.attribute_category_id == category.id } + values_for_these_fields = values_for_this_page.select { |v| fields_for_this_category.map(&:id).include? v.attribute_field_id } + + fields_for_this_category.each do |field| + value_for_this_field = values_for_these_fields.detect do |value| + value.attribute_field_id == field.id && + value.entity_type == page_type.name && + value.entity_id == page.id + end + + case field.field_type + when 'text_area', 'textarea' + # For answers to text fields, we can just output their value directly + if value_for_this_field.try(:value).present? || INCLUDE_EMPTY_FIELD_LABELS + page_object[category.label][field.label] = value_for_this_field.try :value + end + + when 'link' + # For link fields, we have link codes ([[Character-1]]) and need to translate + # them into the linked page's name in order for them to actually be useful + if value_for_this_field.present? + json_list = JSON.parse(value_for_this_field.value) + formatted_name_objects = json_list.map do |link_code| + link_type, link_id = link_code.split('-') + page_ref = query_link_code_with_cache(link_type, link_id) + + name_object = { + name: page_ref.name, + id: page_ref.id, + } + end + + page_object[category.label][field.label] = formatted_name_objects + else + if INCLUDE_EMPTY_FIELD_LABELS + page_object[category.label][field.label] = nil + end + end + end + + end + end + end + + # Append the JSON page object we just built to the list of pages of that kind + export_object[page_type.name].push page_object + end + end + + return export_object.to_json + end + + def self.xml_export(universe_ids) + + end + + def self.yaml_export(universe_ids) + + end + + def self.html_export(universe_ids) + + end + + def self.scrivener_export(universe_ids) + + end + + private + + def self.fetch_categories(user_id:) + AttributeCategory.where(user_id: user_id) + .order('position ASC') + .select(:id, :label, :entity_type) + end + + def self.fetch_fields(category_ids:) + AttributeField.where(attribute_category_id: category_ids) + .order('position ASC') + .select(:id, :label, :attribute_category_id, :field_type) + end + + def self.fetch_values(field_ids:) + Attribute.where(attribute_field_id: field_ids) + .select(:attribute_field_id, :value, :entity_type, :entity_id) + end + + def self.fetch_universes(universe_ids) + + end + + def self.query_link_code_with_cache(content_type_name, content_id) + cache_key = "#{content_type_name}-#{content_id}" + + # Pull from the cache to avoid a query if we can + @content_cache ||= {} + if @content_cache.key?(cache_key) + return @content_cache[cache_key] + end + + # TODO: we should probably whitelist content_type from valid page types here + + # If there's no cache, we unfortunately need to do a query to resolve the link code + content = class_from_name(content_type_name).find(content_id) + @content_cache[cache_key] = content + + return content + end + + def self.class_from_name(content_type_name) + # This lookup is ~3x faster than .constantize + Rails.application.config.content_types_by_name[content_type_name] + end +end \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index e2ddef15..c91b7428 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -155,6 +155,7 @@ Rails.application.routes.draw do get '/', to: 'export#index', as: :notebook_export get '/outline', to: 'export#outline', as: :notebook_outline + get '/markdown', to: 'export#markdown', as: :notebook_markdown get '/notebook.json', to: 'export#notebook_json', as: :notebook_json get '/notebook.xml', to: 'export#notebook_xml', as: :notebook_xml get '/notebook.yml', to: 'export#notebook_yml', as: :notebook_yml diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 536aebc4..7b5f8a6e 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -6,4 +6,5 @@ - default - cache - notifications + - export - low_priority diff --git a/lib/tasks/data_integrity.rake b/lib/tasks/data_integrity.rake index 77a6bd2e..e88221c9 100644 --- a/lib/tasks/data_integrity.rake +++ b/lib/tasks/data_integrity.rake @@ -112,6 +112,23 @@ namespace :data_integrity do end + desc "Remove orphan page references" + task remove_orphan_page_references: :environment do + PageReference.find_each do |reference| + if reference.referencing_page.nil? + puts "Deleting reference #{reference.id}" + reference.destroy + next + end + + if reference.referenced_page.nil? + puts "Deleting reference #{reference.id}" + reference.destroy + next + end + end + end + desc "Ensure all users have the correct upload bandwidth amounts" task correct_bandwidths: :environment do base_bandwidth = User.new.upload_bandwidth_kb # 50_000