paste.elyograg.org

cache handler diff

From elyograg, 4 Months ago, written in Diff-output, viewed 21 times.
URL https://paste.elyograg.org/view/eaca9fb0 Embed
Download Paste or View Raw
  1. diff --git a/solr/bin/solr b/solr/bin/solr
  2. index 704efcb724f..c8b85473949 100755
  3. --- a/solr/bin/solr
  4. +++ b/solr/bin/solr
  5. @@ -85,7 +85,7 @@ done
  6.  
  7.  CDPATH=''  # Prevent "file or directory not found" for 'cdpath' users
  8.  SOLR_TIP=`dirname "$SOLR_SCRIPT"`/..
  9. -SOLR_TIP=`cd "$SOLR_TIP"; pwd`
  10. +SOLR_TIP=`cd "$SOLR_TIP"; pwd -P`
  11.  DEFAULT_SERVER_DIR="$SOLR_TIP/server"
  12.  
  13.  # If an include wasn't specified in the environment, then search for one...
  14. @@ -1502,7 +1502,7 @@ if [[ "$SCRIPT_CMD" == "auth" ]]; then
  15.              fi
  16.  
  17.              if [[ "$2" == "." || "$2" == "./" || "$2" == ".." || "$2" == "../" ]]; then
  18. -              SOLR_SERVER_DIR="$(pwd)/$2"
  19. +              SOLR_SERVER_DIR="$(pwd -P)/$2"
  20.              else
  21.                # see if the arg value is relative to the tip vs full path
  22.                if [[ "$2" != /* ]] && [[ -d "$SOLR_TIP/$2" ]]; then
  23. @@ -1512,7 +1512,7 @@ if [[ "$SCRIPT_CMD" == "auth" ]]; then
  24.                fi
  25.              fi
  26.              # resolve it to an absolute path
  27. -            SOLR_SERVER_DIR="$(cd "$SOLR_SERVER_DIR"; pwd)"
  28. +            SOLR_SERVER_DIR="$(cd "$SOLR_SERVER_DIR"; pwd -P)"
  29.              shift 2
  30.          ;;
  31.          -s|-solr.home)
  32. @@ -1549,8 +1549,8 @@ if [[ "$SCRIPT_CMD" == "auth" ]]; then
  33.    if [ -z "${SOLR_HOME:-}" ]; then
  34.      SOLR_HOME="$SOLR_SERVER_DIR/solr"
  35.    elif [[ $SOLR_HOME != /* ]]; then
  36. -    if [[ -d "`pwd`/$SOLR_HOME" ]]; then
  37. -      SOLR_HOME="$(pwd)/$SOLR_HOME"
  38. +    if [[ -d "`pwd -P`/$SOLR_HOME" ]]; then
  39. +      SOLR_HOME="$(pwd -P)/$SOLR_HOME"
  40.      elif [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then
  41.        SOLR_HOME="$SOLR_SERVER_DIR/$SOLR_HOME"
  42.        SOLR_PID_DIR="$SOLR_HOME"
  43. @@ -1652,7 +1652,7 @@ if [ $# -gt 0 ]; then
  44.              fi
  45.  
  46.              if [[ "$2" == "." || "$2" == "./" || "$2" == ".." || "$2" == "../" ]]; then
  47. -              SOLR_SERVER_DIR="$(pwd)/$2"
  48. +              SOLR_SERVER_DIR="$(pwd -P)/$2"
  49.              else
  50.                # see if the arg value is relative to the tip vs full path
  51.                if [[ "$2" != /* ]] && [[ -d "$SOLR_TIP/$2" ]]; then
  52. @@ -1662,7 +1662,7 @@ if [ $# -gt 0 ]; then
  53.                fi
  54.              fi
  55.              # resolve it to an absolute path
  56. -            SOLR_SERVER_DIR="$(cd "$SOLR_SERVER_DIR"; pwd)"
  57. +            SOLR_SERVER_DIR="$(cd "$SOLR_SERVER_DIR"; pwd -P)"
  58.              shift 2
  59.          ;;
  60.          -s|-solr.home)
  61. @@ -1946,8 +1946,8 @@ fi
  62.  if [ -z "${SOLR_HOME:-}" ]; then
  63.    SOLR_HOME="$SOLR_SERVER_DIR/solr"
  64.  elif [[ $SOLR_HOME != /* ]]; then
  65. -  if [[ -d "`pwd`/$SOLR_HOME" ]]; then
  66. -    SOLR_HOME="$(pwd)/$SOLR_HOME"
  67. +  if [[ -d "`pwd -P`/$SOLR_HOME" ]]; then
  68. +    SOLR_HOME="$(pwd -P)/$SOLR_HOME"
  69.    elif [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then
  70.      SOLR_HOME="$SOLR_SERVER_DIR/$SOLR_HOME"
  71.      SOLR_PID_DIR="$SOLR_HOME"
  72. diff --git a/solr/core/src/java/org/apache/solr/handler/CacheInfoHandler.java b/solr/core/src/java/org/apache/solr/handler/CacheInfoHandler.java
  73. new file mode 100644
  74. index 00000000000..c03e6beab1e
  75. --- /dev/null
  76. +++ b/solr/core/src/java/org/apache/solr/handler/CacheInfoHandler.java
  77. @@ -0,0 +1,99 @@
  78. +package org.apache.solr.handler;
  79. +
  80. +import java.util.ArrayList;
  81. +import java.util.Collections;
  82. +import java.util.Comparator;
  83. +import java.util.HashMap;
  84. +import java.util.LinkedHashMap;
  85. +import java.util.List;
  86. +import java.util.Map;
  87. +import java.util.Map.Entry;
  88. +import java.util.Set;
  89. +
  90. +import org.apache.lucene.search.Query;
  91. +import org.apache.solr.core.SolrCore;
  92. +import org.apache.solr.request.SolrQueryRequest;
  93. +import org.apache.solr.response.SolrQueryResponse;
  94. +import org.apache.solr.search.DocSet;
  95. +import org.apache.solr.search.SolrCache;
  96. +import org.apache.solr.security.AuthorizationContext;
  97. +import org.apache.solr.util.plugin.SolrCoreAware;
  98. +
  99. +/**
  100. + * Handler that returns useful info
  101. + */
  102. +public class CacheInfoHandler extends RequestHandlerBase implements SolrCoreAware {
  103. +  private final static String ACTION = "action";
  104. +  private final static String ACTION_DUMP = "dump";
  105. +  private final static String CACHE_TYPE_FILTER = "filter";
  106. +  private final static String COUNT = "count";
  107. +  private final static String DESCRIPTION = "Returns keys from the specified cache type";
  108. +  private static SolrCore solrCore = null;
  109. +
  110. +  @Override
  111. +  public Name getPermissionName(AuthorizationContext request) {
  112. +    return null;
  113. +  }
  114. +
  115. +  @Override
  116. +  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
  117. +    // TODO Auto-generated method stub
  118. +    String action = req.getParams().get(ACTION, ACTION_DUMP);
  119. +    String type = req.getParams().get(TYPE, CACHE_TYPE_FILTER);
  120. +    @SuppressWarnings("unused")
  121. +    int count = Integer.parseInt(req.getParams().get(COUNT, "-1"));
  122. +
  123. +    switch (action) {
  124. +      case ACTION_DUMP:
  125. +        switch (type) {
  126. +          case CACHE_TYPE_FILTER:
  127. +            // TODO: NOCOMMIT: implement.
  128. +            SolrCache<Query,DocSet> cache = solrCore.getSearcher().get().getFilterCache();
  129. +            Set<Query> keys = cache.keySet();
  130. +            // List<String> queries = new ArrayList<>();
  131. +            Map<String,Long> queryMap = new HashMap<>();
  132. +            Map<String,Long> sortedMap = new LinkedHashMap<>();
  133. +            List<Long> tmpList = new ArrayList<>();
  134. +            for (Query k : keys) {
  135. +              queryMap.put(k.toString(), cache.getHitCount(k));
  136. +            }
  137. +            for (Map.Entry<String,Long> entry : queryMap.entrySet()) {
  138. +              tmpList.add(entry.getValue());
  139. +            }
  140. +            Collections.sort(tmpList, new Comparator<Long>() {
  141. +              public int compare(Long long1, Long long2) {
  142. +                return long2.compareTo(long1);
  143. +              }
  144. +            });
  145. +            for (Long value : tmpList) {
  146. +              for (Entry<String,Long> entry : queryMap.entrySet()) {
  147. +                if (entry.getValue().equals(value)) {
  148. +                  sortedMap.put(entry.getKey(), value);
  149. +                }
  150. +              }
  151. +            }
  152. +            rsp.add("queries", sortedMap);
  153. +            break;
  154. +
  155. +          default:
  156. +            rsp.add("error", "Unknown type: " + type);
  157. +            break;
  158. +        }
  159. +        break;
  160. +
  161. +      default:
  162. +        rsp.add("error", "Unknown action: " + action);
  163. +        break;
  164. +    }
  165. +  }
  166. +
  167. +  @Override
  168. +  public String getDescription() {
  169. +    return DESCRIPTION;
  170. +  }
  171. +
  172. +  @Override
  173. +  public void inform(SolrCore core) {
  174. +    solrCore = core;
  175. +  }
  176. +}
  177. diff --git a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
  178. index e1263a6ed49..4a6bf1a4918 100644
  179. --- a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
  180. +++ b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
  181. @@ -16,14 +16,6 @@
  182.   */
  183.  package org.apache.solr.search;
  184.  
  185. -import com.github.benmanes.caffeine.cache.AsyncCache;
  186. -import com.github.benmanes.caffeine.cache.Cache;
  187. -import com.github.benmanes.caffeine.cache.Caffeine;
  188. -import com.github.benmanes.caffeine.cache.Policy.Eviction;
  189. -import com.github.benmanes.caffeine.cache.RemovalCause;
  190. -import com.github.benmanes.caffeine.cache.RemovalListener;
  191. -import com.github.benmanes.caffeine.cache.stats.CacheStats;
  192. -import com.google.common.annotations.VisibleForTesting;
  193.  import java.io.IOException;
  194.  import java.io.UncheckedIOException;
  195.  import java.lang.invoke.MethodHandles;
  196. @@ -33,22 +25,33 @@ import java.util.Locale;
  197.  import java.util.Map;
  198.  import java.util.Map.Entry;
  199.  import java.util.Optional;
  200. +import java.util.Set;
  201.  import java.util.concurrent.CompletableFuture;
  202.  import java.util.concurrent.CompletionException;
  203. +import java.util.concurrent.ConcurrentHashMap;
  204.  import java.util.concurrent.Executor;
  205.  import java.util.concurrent.ExecutorService;
  206.  import java.util.concurrent.ForkJoinPool;
  207.  import java.util.concurrent.TimeUnit;
  208.  import java.util.concurrent.atomic.LongAdder;
  209. +
  210.  import org.apache.lucene.util.Accountable;
  211.  import org.apache.lucene.util.RamUsageEstimator;
  212. -import org.apache.solr.common.SolrException;
  213.  import org.apache.solr.metrics.MetricsMap;
  214.  import org.apache.solr.metrics.SolrMetricsContext;
  215.  import org.apache.solr.util.IOFunction;
  216.  import org.slf4j.Logger;
  217.  import org.slf4j.LoggerFactory;
  218.  
  219. +import com.github.benmanes.caffeine.cache.AsyncCache;
  220. +import com.github.benmanes.caffeine.cache.Cache;
  221. +import com.github.benmanes.caffeine.cache.Caffeine;
  222. +import com.github.benmanes.caffeine.cache.Policy.Eviction;
  223. +import com.github.benmanes.caffeine.cache.RemovalCause;
  224. +import com.github.benmanes.caffeine.cache.RemovalListener;
  225. +import com.github.benmanes.caffeine.cache.stats.CacheStats;
  226. +import com.google.common.annotations.VisibleForTesting;
  227. +
  228.  /**
  229.   * A SolrCache backed by the Caffeine caching library [1]. By default it uses the Window TinyLFU
  230.   * (W-TinyLFU) eviction policy.
  231. @@ -89,6 +92,7 @@ public class CaffeineCache<K, V> extends SolrCacheBase
  232.    private LongAdder hits;
  233.    private LongAdder inserts;
  234.    private LongAdder lookups;
  235. +  private Map<K,LongAdder> hitCounterMap = null;
  236.    private Cache<K, V> cache;
  237.    private AsyncCache<K, V> asyncCache;
  238.    private long warmupTime;
  239. @@ -98,6 +102,7 @@ public class CaffeineCache<K, V> extends SolrCacheBase
  240.    private int maxIdleTimeSec;
  241.    private boolean cleanupThread;
  242.    private boolean async;
  243. +  private boolean extraStats;
  244.  
  245.    private MetricsMap cacheMap;
  246.    private SolrMetricsContext solrMetricsContext;
  247. @@ -114,6 +119,8 @@ public class CaffeineCache<K, V> extends SolrCacheBase
  248.      super.init(args, regenerator);
  249.      String str = args.get(SIZE_PARAM);
  250.      maxSize = (str == null) ? 1024 : Integer.parseInt(str);
  251. +    str = args.get(EXTRA_STATS_PARAM);
  252. +    extraStats = Boolean.parseBoolean(str);
  253.      str = args.get(INITIAL_SIZE_PARAM);
  254.      initialSize = Math.min((str == null) ? 1024 : Integer.parseInt(str), maxSize);
  255.      str = args.get(MAX_IDLE_TIME_PARAM);
  256. @@ -149,6 +156,10 @@ public class CaffeineCache<K, V> extends SolrCacheBase
  257.              + RamUsageEstimator.shallowSizeOfInstance(executor.getClass())
  258.              + RamUsageEstimator.sizeOfObject(description);
  259.  
  260. +    if (extraStats) {
  261. +      hitCounterMap = new ConcurrentHashMap<>();
  262. +    }
  263. +
  264.      return persistence;
  265.    }
  266.  
  267. @@ -200,7 +211,13 @@ public class CaffeineCache<K, V> extends SolrCacheBase
  268.  
  269.    @Override
  270.    public V get(K key) {
  271. -    return cache.getIfPresent(key);
  272. +    V v = cache.getIfPresent(key);
  273. +    if (extraStats) {
  274. +      if (v != null) {
  275. +        hitCounterMap.get(key).increment();
  276. +      }
  277. +    }
  278. +    return v;
  279.    }
  280.  
  281.    private V computeAsync(K key, IOFunction<? super K, ? extends V> mappingFunction)
  282. @@ -274,6 +291,9 @@ public class CaffeineCache<K, V> extends SolrCacheBase
  283.  
  284.    @Override
  285.    public V put(K key, V val) {
  286. +    if (extraStats) {
  287. +      hitCounterMap.put(key, new LongAdder());
  288. +    }
  289.      inserts.increment();
  290.      V old = cache.asMap().put(key, val);
  291.      recordRamBytes(key, old, val);
  292. @@ -308,6 +328,20 @@ public class CaffeineCache<K, V> extends SolrCacheBase
  293.      return cache.asMap().remove(key);
  294.    }
  295.  
  296. +  @Override
  297. +  public long getHitCount(K key) {
  298. +    long hits = 0;
  299. +    if (hitCounterMap != null && hitCounterMap.containsKey(key)) {
  300. +      hits = hitCounterMap.get(key).sum();
  301. +    }
  302. +    return hits;
  303. +  }
  304. +
  305. +  @Override
  306. +  public Set<K> keySet() {
  307. +    return cache.asMap().keySet();
  308. +  }
  309. +
  310.    @Override
  311.    public void clear() {
  312.      cache.invalidateAll();
  313. @@ -405,7 +439,7 @@ public class CaffeineCache<K, V> extends SolrCacheBase
  314.            break;
  315.          }
  316.        } catch (Exception e) {
  317. -        SolrException.log(log, "Error during auto-warming of key:" + entry.getKey(), e);
  318. +        log.error("Error during auto-warming of key:" + entry.getKey(), e);
  319.        }
  320.      }
  321.  
  322. diff --git a/solr/core/src/java/org/apache/solr/search/SolrCache.java b/solr/core/src/java/org/apache/solr/search/SolrCache.java
  323. index 870d18ea2c9..5ff477833ff 100644
  324. --- a/solr/core/src/java/org/apache/solr/search/SolrCache.java
  325. +++ b/solr/core/src/java/org/apache/solr/search/SolrCache.java
  326. @@ -18,7 +18,9 @@ package org.apache.solr.search;
  327.  
  328.  import java.io.IOException;
  329.  import java.util.Map;
  330. +import java.util.Set;
  331.  import java.util.function.Function;
  332. +
  333.  import org.apache.solr.core.SolrInfoBean;
  334.  import org.apache.solr.util.IOFunction;
  335.  
  336. @@ -36,6 +38,7 @@ public interface SolrCache<K, V> extends SolrInfoBean {
  337.    String MAX_RAM_MB_PARAM = "maxRamMB";
  338.    String MAX_IDLE_TIME_PARAM = "maxIdleTime";
  339.    String INITIAL_SIZE_PARAM = "initialSize";
  340. +  String EXTRA_STATS_PARAM = "extraStats";
  341.    String CLEANUP_THREAD_PARAM = "cleanupThread";
  342.    String ASYNC_PARAM = "async";
  343.  
  344. @@ -89,6 +92,21 @@ public interface SolrCache<K, V> extends SolrInfoBean {
  345.  
  346.    public V remove(K key);
  347.  
  348. +  /**
  349. +   * Gets the number of hits for the provided key.
  350. +   *
  351. +   * @param key The key to look up.
  352. +   * @return The number of hits for the provided key, 0 if the key is not present.
  353. +   */
  354. +  public long getHitCount(K key);
  355. +
  356. +  /**
  357. +   * Gets the keys that the cache contains.
  358. +   *
  359. +   * @return A set of keys found in the cache.
  360. +   */
  361. +  public Set<K> keySet();
  362. +
  363.    /**
  364.     * Get an existing element or atomically compute it if missing.
  365.     *
  366. diff --git a/solr/core/src/resources/ImplicitPlugins.json b/solr/core/src/resources/ImplicitPlugins.json
  367. index b0ee1fec3ca..783cdd03e40 100644
  368. --- a/solr/core/src/resources/ImplicitPlugins.json
  369. +++ b/solr/core/src/resources/ImplicitPlugins.json
  370. @@ -58,6 +58,9 @@
  371.          "q": "{!lucene}*:*"
  372.        }
  373.      },
  374. +    "/admin/cache": {
  375. +      "class": "org.apache.solr.handler.CacheInfoHandler"
  376. +    },
  377.      "/admin/segments": {
  378.        "class": "solr.SegmentsInfoRequestHandler",
  379.        "useParams":"_ADMIN_SEGMENTS"
  380.  

Reply to "cache handler diff"

Here you can reply to the paste above

Powered by Stikked